00000080:00000010:1.1:1788571623.041480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346366b40.
00000080:00000010:1.1:1788571623.041494:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534637eb40.
00000080:00000010:1.1:1788571623.041501:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534637a940.
00000080:00000010:1.1:1788571623.041508:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534637ca40.
00000080:00000010:1.1:1788571623.041513:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534637f380.
00000080:00000010:1.1:1788571623.041518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463798c0.
00000080:00000010:1.1:1788571623.041524:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534637d280.
00000080:00000010:1.1:1788571623.041530:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518f0000.
00000080:00000010:1.1:1788571623.041538:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534637a100.
00000080:00000010:1.1:1788571623.041545:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346368840.
00000080:00000010:1.1:1788571623.212823:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537303b180.
00000080:00000010:1.1:1788571623.212839:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537300f380.
00000080:00000010:1.1:1788571623.212853:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730098c0.
00000080:00000010:1.1:1788571623.212860:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537300d280.
00000080:00000010:1.1:1788571623.212866:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537300a100.
00000080:00000010:1.1:1788571623.212871:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537300b9c0.
00000080:00000010:1.1:1788571623.212878:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537300dac0.
00000080:00000010:1.1:1788571623.212886:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373038840.
00000080:00000010:1.1:1788571623.212896:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953547318c0.
00000080:00000010:1.1:1788571623.212907:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537303eb40.
00000080:00000010:1.1:1788571623.212913:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537303a940.
00000080:00000010:1.1:1788571623.212920:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537303ca40.
00000080:00000010:1.1:1788571623.212925:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537303f380.
00000080:00000010:1.1:1788571623.212932:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730398c0.
00000080:00000010:1.1:1788571623.212938:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537303d280.
00000080:00000010:1.1:1788571623.212943:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537303a100.
00000080:00000010:1.1:1788571623.212949:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537303b9c0.
00000080:00000010:1.1:1788571623.212954:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537303dac0.
00000080:00000010:1.1:1788571623.212959:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373039080.
00000080:00000010:1.1:1788571623.212965:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354735280.
00000080:00000010:1.1:1788571623.212975:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373038000.
00000080:00000010:1.1:1788571623.212982:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730318c0.
00000080:00000010:1.1:1788571623.212990:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373035280.
00000080:00000010:1.1:1788571623.212997:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373032100.
00000080:00000010:1.1:1788571623.213071:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730339c0.
00000080:00000010:1.1:1788571623.213083:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373035ac0.
00000080:00000010:1.1:1788571623.213089:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373031080.
00000080:00000010:1.1:1788571623.213095:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373030000.
00000080:00000010:1.1:1788571623.213101:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373036300.
00000080:00000010:1.1:1788571623.213110:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb5ac0.
00000080:00000010:1.1:1788571623.421063:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534217b9c0.
00000080:00000010:1.1:1788571623.421095:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373052940.
00000080:00000010:1.1:1788571623.492504:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533aeb40.
00000080:00000010:1.1:1788571623.492525:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb2940.
00000080:00000010:1.1:1788571623.525594:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb4a40.
00000080:00000010:1.1:1788571623.525609:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb7380.
00000080:00000010:1.1:1788571623.525632:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533aa940.
00000080:00000010:1.1:1788571623.525654:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533aca40.
00000080:00000010:1.1:1788571623.525666:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534967dac0.
00000080:00000010:1.1:1788571623.525673:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533af380.
00000080:00000010:1.1:1788571623.525679:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a98c0.
00000080:00000010:1.1:1788571623.525688:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b5dac0.
00000080:00000010:1.1:1788571623.525694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b59080.
00000080:00000010:1.1:1788571623.525701:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ad280.
00000080:00000010:1.1:1788571623.525707:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533aa100.
00000080:00000010:1.1:1788571623.525717:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ab9c0.
00000080:00000010:1.1:1788571623.525724:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533adac0.
00000080:00000010:1.1:1788571623.525730:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a9080.
00000080:00000010:1.1:1788571623.525736:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a8000.
00000080:00000010:1.1:1788571623.525742:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349679080.
00000080:00000010:1.1:1788571623.525748:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bbc200.
00000080:00000010:1.1:1788571623.598618:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba8840.
00000080:00000010:1.1:1788571623.598647:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351baeb40.
00000080:00000010:1.1:1788571623.598657:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351baa940.
00000080:00000010:1.1:1788571623.598663:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351baca40.
00000080:00000010:1.1:1788571623.718450:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b42100.
00000080:00000010:1.1:1788571623.718470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b439c0.
00000080:00000010:1.1:1788571623.718477:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b45ac0.
00000080:00000010:1.1:1788571623.718484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b41080.
00000080:00000010:1.1:1788571623.718491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b40000.
00000080:00000010:1.1:1788571623.718502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b46300.
00000080:00000010:1.1:1788571623.718508:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b44200.
00000080:00000010:1.1:1788571623.718515:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b43180.
00000080:00000010:1.1:1788571623.718521:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b40840.
00000080:00000010:1.1:1788571623.718530:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d0000.
00000080:00000010:1.1:1788571623.718538:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518d6300.
00000080:00000010:1.1:1788571623.718547:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351914200.
00000080:00000010:1.1:1788571623.718555:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544fdac0.
00000080:00000010:1.1:1788571623.718562:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f9080.
00000080:00000010:1.1:1788571623.718569:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f8000.
00000080:00000010:1.1:1788571623.718574:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544fe300.
00000080:00000010:1.1:1788571623.718580:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544fc200.
00000080:00000010:1.1:1788571623.718585:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544fb180.
00000080:00000010:1.1:1788571623.718593:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f5ac0.
00000080:00000010:1.1:1788571623.718600:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f8840.
00000080:00000010:1.1:1788571623.718606:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544feb40.
00000080:00000010:1.1:1788571623.718612:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544fa940.
00000080:00000010:1.1:1788571623.718617:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d6300.
00000080:00000010:1.1:1788571623.751610:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544eb9c0.
00000080:00000010:1.1:1788571623.751650:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544edac0.
00000080:00000010:1.1:1788571623.751657:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544e9080.
00000080:00000010:1.1:1788571623.751664:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544e8000.
00000080:00000010:1.1:1788571623.751669:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544ee300.
00000080:00000010:1.1:1788571623.751674:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544ec200.
00000080:00000010:1.1:1788571623.751684:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544eb180.
00000080:00000010:1.1:1788571623.902466:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349743180.
00000080:00000010:1.1:1788571623.902489:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d0840.
00000080:00000010:1.1:1788571623.902504:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349740840.
00000080:00000010:1.1:1788571623.902513:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349746b40.
00000080:00000010:1.1:1788571623.902521:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349742940.
00000080:00000010:1.1:1788571623.902532:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349744a40.
00000080:00000010:1.1:1788571623.902538:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349747380.
00000080:00000010:1.1:1788571623.902549:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497498c0.
00000080:00000010:1.1:1788571623.902556:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534974d280.
00000080:00000010:1.1:1788571623.902561:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534974a100.
00000080:00000010:1.1:1788571623.902567:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534974b9c0.
00000080:00000010:1.1:1788571623.902577:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346338840.
00000080:00000010:1.1:1788571623.902598:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534217f380.
00000080:00000010:1.1:1788571623.902619:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518d0000.
00000080:00000010:1.1:1788571623.902642:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534633eb40.
00000080:00000010:1.1:1788571623.902650:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534633a940.
00000080:00000010:1.1:1788571623.902657:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534633ca40.
00000080:00000010:1.1:1788571623.902664:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534974dac0.
00000080:00000010:1.1:1788571623.902671:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497418c0.
00000080:00000010:1.1:1788571623.902677:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349745280.
00000080:00000010:1.1:1788571623.902686:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349749080.
00000080:00000010:1.1:1788571623.902692:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349742100.
00000080:00000010:1.1:1788571623.902697:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497439c0.
00000080:00000010:1.1:1788571623.902703:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463318c0.
00000080:00000010:1.1:1788571623.902783:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d6b40.
00000080:00000010:1.1:1788571623.902794:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346335280.
00000080:00000010:1.1:1788571623.902800:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346332100.
00000080:00000010:1.1:1788571623.902807:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463339c0.
00000080:00000010:1.1:1788571623.902815:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346380000.
00000080:00000010:1.1:1788571623.902822:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346386300.
00000080:00000010:1.1:1788571623.902829:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346384200.
00000080:00000010:1.1:1788571623.902835:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346383180.
00000080:00000010:1.1:1788571623.902841:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346335ac0.
00000080:00000010:1.1:1788571623.902847:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346331080.
00000080:00000010:1.1:1788571623.902853:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346330000.
00000080:00000010:1.1:1788571624.050555:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e5280.
00000080:00000010:1.1:1788571624.050570:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e2100.
00000080:00000010:1.1:1788571624.050583:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544939c0.
00000080:00000010:1.1:1788571624.050590:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3a100.
00000080:00000010:1.1:1788571624.050597:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354495ac0.
00000080:00000010:1.1:1788571624.050604:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354491080.
00000080:00000010:1.1:1788571624.050610:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354490000.
00000080:00000010:1.1:1788571624.050615:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354496300.
00000080:00000010:1.1:1788571624.050621:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354494200.
00000080:00000010:1.1:1788571624.050637:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354493180.
00000080:00000010:1.1:1788571624.050645:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535447ca40.
00000080:00000010:1.1:1788571624.050653:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535447f380.
00000080:00000010:1.1:1788571624.149070:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354490840.
00000080:00000010:1.1:1788571624.149094:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354496b40.
00000080:00000010:1.1:1788571624.149137:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537307b9c0.
00000080:00000010:1.1:1788571624.149149:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354492940.
00000080:00000010:1.1:1788571624.149157:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354500000.
00000080:00000010:1.1:1788571624.149167:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354506300.
00000080:00000010:1.1:1788571624.149174:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354504200.
00000080:00000010:1.1:1788571624.149183:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354503180.
00000080:00000010:1.1:1788571624.149189:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354500840.
00000080:00000010:1.1:1788571624.149196:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354506b40.
00000080:00000010:1.1:1788571624.149203:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354502940.
00000080:00000010:1.1:1788571624.149210:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354504a40.
00000080:00000010:1.1:1788571624.149332:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354507380.
00000080:00000010:1.1:1788571624.149342:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89537307ca40.
00000080:00000010:1.1:1788571624.149350:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953545018c0.
00000080:00000010:1.1:1788571624.149359:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354505280.
00000080:00000010:1.1:1788571624.149367:0:20:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354502100.
00000080:00000010:1.1:1788571624.229496:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534210a940.
00000080:00000010:1.1:1788571624.229510:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534210ca40.
00000080:00000010:1.1:1788571624.229517:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534210f380.
00000080:00000010:1.1:1788571624.229531:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421098c0.
00000080:00000010:1.1:1788571624.229538:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535451a100.
00000080:00000010:1.1:1788571624.229547:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352638000.
00000080:00000010:1.1:1788571624.229556:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532d2100.
00000080:00000010:1.1:1788571624.229564:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351912940.
00000080:00000010:1.1:1788571624.229572:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535263e300.
00000080:00000010:1.1:1788571624.318520:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354748840.
00000080:00000010:1.1:1788571624.318569:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420bb9c0.
00000080:00000010:1.1:1788571624.353720:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953524d0840.
00000080:00000010:1.1:1788571624.353732:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953524d7380.
00000080:00000010:1.1:1788571624.353744:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953524d4a40.
00000080:00000010:1.1:1788571624.353756:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420bdac0.
00000080:00000010:1.1:1788571624.353763:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420b9080.
00000080:00000010:1.1:1788571624.353770:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420b8000.
00000080:00000010:1.1:1788571624.353775:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420be300.
00000080:00000010:1.1:1788571624.420451:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526ae300.
00000080:00000010:1.1:1788571624.420486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526ac200.
00000080:00000010:1.1:1788571624.420495:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474c200.
00000080:00000010:1.1:1788571624.420506:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526ab180.
00000080:00000010:1.1:1788571624.420512:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526a8840.
00000080:00000010:1.1:1788571624.420518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526aeb40.
00000080:00000010:1.1:1788571624.420524:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526aa940.
00000080:00000010:1.1:1788571624.420530:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526aca40.
00000080:00000010:1.1:1788571624.420536:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526af380.
00000080:00000010:1.1:1788571624.420542:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526a98c0.
00000080:00000010:1.1:1788571624.420549:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526ad280.
00000080:00000010:1.1:1788571624.420554:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526aa100.
00000080:00000010:1.1:1788571624.420560:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526ab9c0.
00000080:00000010:1.1:1788571624.420564:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474b180.
00000080:00000010:1.1:1788571624.420574:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526adac0.
00000080:00000010:1.1:1788571624.420583:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526a9080.
00000080:00000010:1.1:1788571624.420589:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526a8000.
00000080:00000010:1.1:1788571624.420595:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e7380.
00000080:00000010:1.1:1788571624.420602:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e18c0.
00000080:00000010:1.1:1788571624.420609:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e5280.
00000080:00000010:1.1:1788571624.420614:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e2100.
00000080:00000010:1.1:1788571624.520651:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342111080.
00000080:00000010:1.1:1788571624.520660:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730e4200.
00000080:00000010:1.1:1788571624.520667:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460ef380.
00000080:00000010:1.1:1788571624.520675:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730e8000.
00000080:00000010:1.1:1788571624.520699:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730ee300.
00000080:00000010:1.1:1788571624.548448:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730e3180.
00000080:00000010:1.1:1788571624.548467:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373096b40.
00000080:00000010:1.1:1788571624.548483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730dd280.
00000080:00000010:1.1:1788571624.548490:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730da100.
00000080:00000010:1.1:1788571624.548496:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730db9c0.
00000080:00000010:1.1:1788571624.548501:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730ddac0.
00000080:00000010:1.1:1788571624.548511:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342110000.
00000080:00000010:1.1:1788571624.548518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d9080.
00000080:00000010:1.1:1788571624.548524:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d8000.
00000080:00000010:1.1:1788571624.548529:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730de300.
00000080:00000010:1.1:1788571624.548534:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730dc200.
00000080:00000010:1.1:1788571624.548540:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730db180.
00000080:00000010:1.1:1788571624.548549:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d8840.
00000080:00000010:1.1:1788571624.548554:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730deb40.
00000080:00000010:1.1:1788571624.548559:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730da940.
00000080:00000010:1.1:1788571624.548565:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730dca40.
00000080:00000010:1.1:1788571624.548570:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730df380.
00000080:00000010:1.1:1788571624.548575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342116300.
00000080:00000010:1.1:1788571624.548580:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d98c0.
00000080:00000010:1.1:1788571624.548586:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d7380.
00000080:00000010:1.1:1788571624.548670:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d18c0.
00000080:00000010:1.1:1788571624.548689:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d5280.
00000080:00000010:1.1:1788571624.548705:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d2100.
00000080:00000010:1.1:1788571624.548716:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d39c0.
00000080:00000010:1.1:1788571624.548724:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d5ac0.
00000080:00000010:1.1:1788571624.548731:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d1080.
00000080:00000010:1.1:1788571624.548737:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d0000.
00000080:00000010:1.1:1788571624.548743:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d6300.
00000080:00000010:1.1:1788571624.548749:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342114200.
00000080:00000010:1.1:1788571624.548755:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d4200.
00000080:00000010:1.1:1788571624.548760:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d3180.
00000080:00000010:1.1:1788571624.548765:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d0840.
00000080:00000010:1.1:1788571624.548770:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d6b40.
00000080:00000010:1.1:1788571624.548774:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d2940.
00000080:00000010:1.1:1788571624.548782:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730d4a40.
00000080:00000010:1.1:1788571624.548788:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373092940.
00000080:00000010:1.1:1788571624.548796:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730cb180.
00000080:00000010:1.1:1788571624.548802:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730c8840.
00000080:00000010:1.1:1788571624.548809:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730ceb40.
00000080:00000010:1.1:1788571624.548817:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342113180.
00000080:00000010:1.1:1788571624.548824:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730ca940.
00000080:00000010:1.1:1788571624.548830:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730cca40.
00000080:00000010:1.1:1788571624.548836:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730cf380.
00000080:00000010:1.1:1788571624.548842:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730c98c0.
00000080:00000010:1.1:1788571624.548848:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730cd280.
00000080:00000010:1.1:1788571624.548854:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730ca100.
00000080:00000010:1.1:1788571624.548863:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730cb9c0.
00000080:00000010:1.1:1788571624.548869:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730cdac0.
00000080:00000010:1.1:1788571624.548875:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373094a40.
00000080:00000010:1.1:1788571624.548882:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895373097380.
00000080:00000010:1.1:1788571624.548888:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342110840.
00000080:00000010:1.1:1788571624.548894:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730c9080.
00000080:00000010:1.1:1788571624.548900:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730c8000.
00000080:00000010:1.1:1788571624.548905:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953730ce300.
00000080:00000010:1.1:1788571624.727445:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460e2100.
00000080:00000010:1.1:1788571624.727462:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460e39c0.
00000080:00000010:1.1:1788571624.727473:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546deb40.
00000080:00000010:1.1:1788571624.727484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460e5ac0.
00000080:00000010:1.1:1788571624.727492:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460d8000.
00000080:00000010:1.1:1788571624.727499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460de300.
00000080:00000010:1.1:1788571624.749466:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460dc200.
00000080:00000010:1.1:1788571624.749483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460db180.
00000080:00000010:1.1:1788571624.749493:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460d8840.
00000080:00000010:1.1:1788571624.749499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460deb40.
00000080:00000010:1.1:1788571624.749506:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460da940.
00000080:00000010:1.1:1788571624.749512:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460dca40.
00000080:00000010:1.1:1788571624.749521:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346356300.
00000080:00000010:1.1:1788571624.749528:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546da940.
00000080:00000010:1.1:1788571624.749537:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460ceb40.
00000080:00000010:1.1:1788571624.749545:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460ca940.
00000080:00000010:1.1:1788571624.749556:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460cca40.
00000080:00000010:1.1:1788571624.749562:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460cf380.
00000080:00000010:1.1:1788571624.749568:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460df380.
00000080:00000010:1.1:1788571624.749578:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460d98c0.
00000080:00000010:1.1:1788571624.749584:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460dd280.
00000080:00000010:1.1:1788571624.749591:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460da100.
00000080:00000010:1.1:1788571624.749599:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460b0840.
00000080:00000010:1.1:1788571624.749606:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460b6b40.
00000080:00000010:1.1:1788571624.749625:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546dca40.
00000080:00000010:1.1:1788571624.749632:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460b2940.
00000080:00000010:1.1:1788571624.749639:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460db9c0.
00000080:00000010:1.1:1788571624.749646:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460ddac0.
00000080:00000010:1.1:1788571624.749653:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460d9080.
00000080:00000010:1.1:1788571624.749659:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460d6300.
00000080:00000010:1.1:1788571624.749665:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460d4200.
00000080:00000010:1.1:1788571624.749671:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460d3180.
00010000:00000010:1.1:1788571625.269428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67800.
00010000:00000010:1.1:1788571647.980565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00010000:00000010:1.1:1788571647.980580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044b00.
00010000:00000010:1.1:1788571647.980584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48a00.
00000080:00000010:1.1:1788571648.003062:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649d40.
00000080:00000010:1.1:1788571648.003080:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649840.
00010000:00000010:1.1:1788571648.037405:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044200.
00010000:00000010:1.1:1788571648.037422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00000080:00000010:1.1:1788571648.042429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649840.
00010000:00000010:1.1:1788571648.059507:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044200.
00010000:00000010:1.1:1788571648.059523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49a00.
00010000:00000010:1.1:1788571665.344629:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044100.
00010000:00000010:1.1:1788571665.344654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48800.
00000080:00000010:1.1:1788571665.384478:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620b9c0.
00000080:00000010:1.1:1788571665.384494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649a00.
00000080:00000010:1.1:1788571687.763509:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349658840.
00000080:00000010:1.1:1788571687.773449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218960.
00000080:00000010:1.1:1788571687.773468:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965eb40.
00000080:00000010:1.1:1788571687.773473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218ec0.
00000080:00000010:1.1:1788571687.773479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965a940.
00000080:00000010:1.1:1788571687.773482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d2188e0.
00000080:00000010:1.1:1788571687.773487:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965ca40.
00000080:00000010:1.1:1788571687.773490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218180.
00000080:00000010:1.1:1788571687.773495:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965f380.
00000080:00000010:1.1:1788571687.773498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218c80.
00000080:00000010:1.1:1788571687.773502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953496598c0.
00000080:00000010:1.1:1788571687.773505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218900.
00000080:00000010:1.1:1788571687.773514:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965d280.
00000080:00000010:1.1:1788571687.773517:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218c40.
00000080:00000010:1.1:1788571687.773525:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965a100.
00000080:00000010:1.1:1788571687.773528:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218d00.
00000080:00000010:1.1:1788571687.773536:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965b9c0.
00000080:00000010:1.1:1788571687.773539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218f00.
00000080:00000010:1.1:1788571687.773544:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965dac0.
00000080:00000010:1.1:1788571687.773547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218cc0.
00000080:00000010:1.1:1788571687.773551:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349659080.
00000080:00000010:1.1:1788571687.773555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655450c0.
00000080:00000010:1.1:1788571687.773604:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349658000.
00000080:00000010:1.1:1788571687.773608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545b80.
00000080:00000010:1.1:1788571687.773614:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965e300.
00000080:00000010:1.1:1788571687.773616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545b20.
00000080:00000010:1.1:1788571687.773621:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965c200.
00000080:00000010:1.1:1788571687.773623:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545f80.
00000080:00000010:1.1:1788571687.773629:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965b180.
00000080:00000010:1.1:1788571687.773631:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545a00.
00000080:00000010:1.1:1788571687.773637:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342110840.
00000080:00000010:1.1:1788571687.773641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545460.
00000080:00000010:1.1:1788571687.773645:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342116b40.
00000080:00000010:1.1:1788571687.773648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545fc0.
00000080:00000010:1.1:1788571687.773657:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342112940.
00000080:00000010:1.1:1788571687.773660:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545de0.
00000080:00000010:1.1:1788571687.773666:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342114a40.
00000080:00000010:1.1:1788571687.773669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545540.
00000080:00000010:1.1:1788571687.773673:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342117380.
00000080:00000010:1.1:1788571687.773675:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545080.
00000080:00000010:1.1:1788571687.773685:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421118c0.
00000080:00000010:1.1:1788571687.773696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655457e0.
00000080:00000010:1.1:1788571687.773701:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342115280.
00000080:00000010:1.1:1788571687.773705:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545140.
00000080:00000010:1.1:1788571687.773710:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342112100.
00000080:00000010:1.1:1788571687.773712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545200.
00000080:00000010:1.1:1788571687.773718:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421139c0.
00000080:00000010:1.1:1788571687.773720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655452e0.
00000080:00000010:1.1:1788571687.773726:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e18c0.
00000080:00000010:1.1:1788571687.773730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545380.
00000080:00000010:1.1:1788571687.773736:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e5280.
00000080:00000010:1.1:1788571687.773739:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545860.
00000080:00000010:1.1:1788571687.773744:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e2100.
00000080:00000010:1.1:1788571687.773747:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d2180c0.
00000080:00000010:1.1:1788571687.773756:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342098000.
00000080:00000010:1.1:1788571687.773759:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218600.
00000080:00000010:1.1:1788571687.773763:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534209e300.
00000080:00000010:1.1:1788571687.773765:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218b60.
00000080:00000010:1.1:1788571687.773770:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e39c0.
00000080:00000010:1.1:1788571687.773774:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8ca20.
00000080:00000010:1.1:1788571687.773779:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e5ac0.
00000080:00000010:1.1:1788571687.773781:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c920.
00000080:00000010:1.1:1788571687.773785:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e1080.
00000080:00000010:1.1:1788571687.773787:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cca0.
00000080:00000010:1.1:1788571687.773792:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e0000.
00000080:00000010:1.1:1788571687.773794:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c8c0.
00000080:00000010:1.1:1788571687.773799:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e6300.
00000080:00000010:1.1:1788571687.773802:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218da0.
00000080:00000010:1.1:1788571687.773806:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e4200.
00000080:00000010:1.1:1788571687.773808:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218e80.
00000080:00000010:1.1:1788571687.773813:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e3180.
00000080:00000010:1.1:1788571687.773815:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218a20.
00000080:00000010:1.1:1788571687.773820:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e0840.
00000080:00000010:1.1:1788571687.773822:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218140.
00000080:00000010:1.1:1788571687.773827:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e6b40.
00000080:00000010:1.1:1788571687.773830:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218b20.
00000080:00000010:1.1:1788571687.773834:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e2940.
00000080:00000010:1.1:1788571687.773836:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d2186e0.
00000080:00000010:1.1:1788571687.773841:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e4a40.
00000080:00000010:1.1:1788571687.773843:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218580.
00000080:00000010:1.1:1788571687.773848:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e7380.
00000080:00000010:1.1:1788571687.773851:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218400.
00000080:00000010:1.1:1788571687.773855:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353235280.
00000080:00000010:1.1:1788571687.773858:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895342e59440.
00000080:00000010:1.1:1788571687.773863:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353232100.
00000080:00000010:1.1:1788571687.773866:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d2183e0.
00000080:00000010:1.1:1788571687.773870:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532339c0.
00000080:00000010:1.1:1788571687.773875:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895353a87e60.
00000080:00000010:1.1:1788571687.773881:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353235ac0.
00000080:00000010:1.1:1788571687.773883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895353a87100.
00000080:00000010:1.1:1788571687.773890:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e3180.
00000080:00000010:1.1:1788571687.773893:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895353a87880.
00000080:00000010:1.1:1788571687.773898:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e0840.
00000080:00000010:1.1:1788571687.773900:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895353a87dc0.
00000080:00000010:1.1:1788571687.773905:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e6b40.
00000080:00000010:1.1:1788571687.773908:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895353a87be0.
00000080:00000010:1.1:1788571687.773912:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e2940.
00000080:00000010:1.1:1788571687.773916:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649900.
00000080:00000010:1.1:1788571687.773921:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e4a40.
00000080:00000010:1.1:1788571687.773925:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649480.
00000080:00000010:1.1:1788571687.773930:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e7380.
00000080:00000010:1.1:1788571687.773933:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655459e0.
00000080:00000010:1.1:1788571687.773937:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e18c0.
00000080:00000010:1.1:1788571687.773940:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545720.
00000080:00000010:1.1:1788571688.071807:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346228840.
00000080:00000010:1.1:1788571688.071824:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:1.1:1788571688.071839:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622e300.
00000080:00000010:1.1:1788571688.071845:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb460.
00000080:00000010:1.1:1788571688.071854:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346229080.
00000080:00000010:1.1:1788571688.071857:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:1.1:1788571688.071863:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622b9c0.
00000080:00000010:1.1:1788571688.071866:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:1.1:1788571688.071873:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622dac0.
00000080:00000010:1.1:1788571688.071876:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8a0.
00000080:00000010:1.1:1788571688.071891:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535337d280.
00000080:00000010:1.1:1788571688.071913:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf20.
00000080:00000010:1.1:1788571688.071921:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349763180.
00000080:00000010:1.1:1788571688.071924:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb540.
00000080:00000010:1.1:1788571688.071930:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349760840.
00000080:00000010:1.1:1788571688.071934:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:1.1:1788571688.071939:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349766b40.
00000080:00000010:1.1:1788571688.071943:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:1.1:1788571688.071948:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947e300.
00000080:00000010:1.1:1788571688.071952:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb360.
00000080:00000010:1.1:1788571688.071956:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947c200.
00000080:00000010:1.1:1788571688.071959:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:1.1:1788571688.071963:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947b180.
00000080:00000010:1.1:1788571688.071966:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf80.
00000080:00000010:1.1:1788571688.071971:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349478840.
00000080:00000010:1.1:1788571688.071975:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d153260.
00000080:00000010:1.1:1788571688.071980:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947eb40.
00000080:00000010:1.1:1788571688.071983:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00000080:00000010:1.1:1788571688.071988:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947a940.
00000080:00000010:1.1:1788571688.071991:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:1.1:1788571688.071994:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947ca40.
00000080:00000010:1.1:1788571688.071997:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb940.
00000080:00000010:1.1:1788571688.072003:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947f380.
00000080:00000010:1.1:1788571688.072014:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb200.
00000080:00000010:1.1:1788571688.072023:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349762940.
00000080:00000010:1.1:1788571688.072025:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb00.
00000080:00000010:1.1:1788571688.072035:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349764a40.
00000080:00000010:1.1:1788571688.072039:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb640.
00000080:00000010:1.1:1788571688.072046:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349767380.
00000080:00000010:1.1:1788571688.072049:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb080.
00000080:00000010:1.1:1788571688.072055:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497618c0.
00000080:00000010:1.1:1788571688.072058:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc00.
00000080:00000010:1.1:1788571688.072070:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349765280.
00000080:00000010:1.1:1788571688.095443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:1.1:1788571688.095459:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349762100.
00000080:00000010:1.1:1788571688.095464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe20.
00000080:00000010:1.1:1788571688.095471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497639c0.
00000080:00000010:1.1:1788571688.095474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0a0.
00000080:00000010:1.1:1788571688.095479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349765ac0.
00000080:00000010:1.1:1788571688.095483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb800.
00000080:00000010:1.1:1788571688.095488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349761080.
00000080:00000010:1.1:1788571688.095491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb860.
00000080:00000010:1.1:1788571688.095498:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349760000.
00000080:00000010:1.1:1788571688.095501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:1.1:1788571688.095507:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349766300.
00000080:00000010:1.1:1788571688.095510:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d153940.
00000080:00000010:1.1:1788571688.095515:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349764200.
00010000:00000010:1.1:1788571688.329699:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e800.
00010000:00000010:1.1:1788571688.329710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372699200.
00010000:00000010:1.1:1788571688.329716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379628a00.
00010000:00000010:1.1:1788571688.329721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044a00.
00010000:00000010:1.1:1788571688.329725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856f800.
00010000:00000010:1.1:1788571688.329728:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044c00.
00010000:00000010:1.1:1788571688.329733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856e400.
00010000:00000010:1.1:1788571688.329736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f4400.
00010000:00000010:1.1:1788571688.329739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f4200.
00010000:00000010:1.1:1788571688.329743:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384a00.
00010000:00000010:1.1:1788571688.329747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abd800.
00010000:00000010:1.1:1788571688.329751:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384000.
00010000:00000010:1.1:1788571688.329756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abc400.
00010000:00000010:1.1:1788571688.329760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00010000:00000010:1.1:1788571688.329762:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384b00.
00010000:00000010:1.1:1788571688.329765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abd600.
00010000:00000010:1.1:1788571688.329769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52c00.
00010000:00000010:1.1:1788571688.329772:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384300.
00010000:00000010:1.1:1788571688.329775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abdc00.
00010000:00000010:1.1:1788571688.329777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00010000:00000010:1.1:1788571688.329780:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384f00.
00010000:00000010:1.1:1788571688.329783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abc200.
00010000:00000010:1.1:1788571688.329785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00010000:00000010:1.1:1788571688.329788:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384400.
00010000:00000010:1.1:1788571688.329790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abc000.
00010000:00000010:1.1:1788571688.329793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a600.
00010000:00000010:1.1:1788571688.329796:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384600.
00010000:00000010:1.1:1788571688.329799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abde00.
00010000:00000010:1.1:1788571688.329801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ae00.
00010000:00000010:1.1:1788571688.329804:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384e00.
00010000:00000010:1.1:1788571688.329807:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abda00.
00010000:00000010:1.1:1788571688.329809:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384c00.
00010000:00000010:1.1:1788571688.329812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abcc00.
00010000:00000010:1.1:1788571688.329816:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325500.
00010000:00000010:1.1:1788571688.329819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abd200.
00010000:00000010:1.1:1788571688.329822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00010000:00000010:1.1:1788571688.329824:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325700.
00010000:00000010:1.1:1788571688.329827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abc800.
00010000:00000010:1.1:1788571688.329829:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325400.
00010000:00000010:1.1:1788571688.329831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abd400.
00010000:00000010:1.1:1788571688.334533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325b00.
00010000:00000010:1.1:1788571688.334549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abca00.
00010000:00000010:1.1:1788571688.334563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325f00.
00010000:00000010:1.1:1788571688.334566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abc600.
00010000:00000010:1.1:1788571688.334570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325300.
00010000:00000010:1.1:1788571688.334573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abce00.
00010000:00000010:1.1:1788571688.334576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325000.
00010000:00000010:1.1:1788571688.334579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344abd000.
00010000:00000010:1.1:1788571688.334581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325a00.
00010000:00000010:1.1:1788571688.334594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34b400.
00010000:00000010:1.1:1788571688.334602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:1.1:1788571688.334606:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325200.
00010000:00000010:1.1:1788571688.334610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656df600.
00010000:00000010:1.1:1788571688.334613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:1.1:1788571688.334615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325900.
00010000:00000010:1.1:1788571688.334619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656de400.
00010000:00000010:1.1:1788571688.334622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00010000:00000010:1.1:1788571688.334624:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325600.
00010000:00000010:1.1:1788571688.334627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656df800.
00010000:00000010:1.1:1788571688.334630:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b300.
00010000:00000010:1.1:1788571688.334633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656dec00.
00010000:00000010:1.1:1788571688.334636:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b900.
00010000:00000010:1.1:1788571688.334638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656df200.
00010000:00000010:1.1:1788571688.334641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:1.1:1788571688.334644:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b800.
00010000:00000010:1.1:1788571688.334647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656de800.
00010000:00000010:1.1:1788571688.334651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199e00.
00010000:00000010:1.1:1788571688.334654:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4bf00.
00010000:00000010:1.1:1788571688.334657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656df400.
00010000:00000010:1.1:1788571688.334661:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528caa00.
00010000:00000010:1.1:1788571688.334664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4bd00.
00010000:00000010:1.1:1788571688.334667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656dea00.
00010000:00000010:1.1:1788571688.334672:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355a00.
00010000:00000010:1.1:1788571688.334675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656de600.
00010000:00000010:1.1:1788571688.334678:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355c00.
00010000:00000010:1.1:1788571688.334680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656dee00.
00010000:00000010:1.1:1788571688.334683:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355500.
00010000:00000010:1.1:1788571688.334685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656df000.
00010000:00000010:1.1:1788571688.334689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08cc00.
00010000:00000010:1.1:1788571688.334692:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355200.
00010000:00000010:1.1:1788571688.334694:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656dfa00.
00010000:00000010:1.1:1788571688.334698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198000.
00010000:00000010:1.1:1788571688.334701:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355000.
00010000:00000010:1.1:1788571688.334703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656dfe00.
00010000:00000010:1.1:1788571688.334706:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355100.
00010000:00000010:1.1:1788571688.337440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656de000.
00010000:00000010:1.1:1788571688.337462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355300.
00010000:00000010:1.1:1788571688.337466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656de200.
00010000:00000010:1.1:1788571688.337469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355900.
00010000:00000010:1.1:1788571688.337472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656dfc00.
00010000:00000010:1.1:1788571688.337477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198200.
00010000:00000010:1.1:1788571688.337480:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355800.
00010000:00000010:1.1:1788571688.337484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c800.
00010000:00000010:1.1:1788571688.337490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199c00.
00010000:00000010:1.1:1788571688.337493:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355f00.
00010000:00000010:1.1:1788571688.337497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d400.
00010000:00000010:1.1:1788571688.337501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00010000:00000010:1.1:1788571688.337505:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325d00.
00010000:00000010:1.1:1788571688.337508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ca00.
00010000:00000010:1.1:1788571688.337510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199600.
00010000:00000010:1.1:1788571688.337515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48400.
00010000:00000010:1.1:1788571688.337519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c600.
00010000:00000010:1.1:1788571688.337522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198400.
00010000:00000010:1.1:1788571688.337525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48600.
00010000:00000010:1.1:1788571688.337528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ce00.
00010000:00000010:1.1:1788571688.337531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199800.
00010000:00000010:1.1:1788571688.337534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48700.
00010000:00000010:1.1:1788571688.337537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d000.
00010000:00000010:1.1:1788571688.337540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198c00.
00010000:00000010:1.1:1788571688.337543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48b00.
00010000:00000010:1.1:1788571688.337546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34da00.
00010000:00000010:1.1:1788571688.337549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199200.
00010000:00000010:1.1:1788571688.337561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48a00.
00010000:00000010:1.1:1788571688.337565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34de00.
00010000:00000010:1.1:1788571688.337567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48300.
00010000:00000010:1.1:1788571688.337570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c000.
00010000:00000010:1.1:1788571688.337573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48800.
00010000:00000010:1.1:1788571688.337576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c200.
00010000:00000010:1.1:1788571688.337579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48d00.
00010000:00000010:1.1:1788571688.337582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34dc00.
00010000:00000010:1.1:1788571688.337593:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48c00.
00010000:00000010:1.1:1788571688.337597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d600.
00010000:00000010:1.1:1788571688.337600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48500.
00010000:00000010:1.1:1788571688.337603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c400.
00010000:00000010:1.1:1788571688.337606:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48e00.
00010000:00000010:1.1:1788571688.337609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d800.
00010000:00000010:1.1:1788571688.337612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198800.
00010000:00000010:1.1:1788571688.337615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48900.
00010000:00000010:1.1:1788571688.337618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34cc00.
00010000:00000010:1.1:1788571688.337621:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48000.
00010000:00000010:1.1:1788571688.337624:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d200.
00010000:00000010:1.1:1788571688.337626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199400.
00010000:00000010:1.1:1788571688.337630:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2b00.
00010000:00000010:1.1:1788571688.337636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08d200.
00010000:00000010:1.1:1788571688.337639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198a00.
00010000:00000010:1.1:1788571688.337642:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2700.
00010000:00000010:1.1:1788571688.337646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08c800.
00010000:00000010:1.1:1788571688.337649:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2600.
00010000:00000010:1.1:1788571688.337652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08d400.
00010000:00000010:1.1:1788571688.337658:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf600.
00010000:00000010:1.1:1788571688.337663:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08ca00.
00010000:00000010:1.1:1788571688.337666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98e00.
00010000:00000010:1.1:1788571688.337668:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48200.
00010000:00000010:1.1:1788571688.337671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08c600.
00010000:00000010:1.1:1788571688.337674:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48f00.
00010000:00000010:1.1:1788571688.337677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08ce00.
00010000:00000010:1.1:1788571688.337680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00010000:00000010:1.1:1788571688.337686:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545af00.
00010000:00000010:1.1:1788571688.337689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08d000.
00010000:00000010:1.1:1788571688.337692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00010000:00000010:1.1:1788571688.337695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545ae00.
00010000:00000010:1.1:1788571688.337698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08da00.
00010000:00000010:1.1:1788571688.337702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355d00.
00010000:00000010:1.1:1788571688.337705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08de00.
00010000:00000010:1.1:1788571688.337708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355700.
00010000:00000010:1.1:1788571688.337712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08c000.
00010000:00000010:1.1:1788571688.337715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198600.
00010000:00000010:1.1:1788571688.337718:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355400.
00010000:00000010:1.1:1788571688.337721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08c200.
00010000:00000010:1.1:1788571688.337724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198e00.
00010000:00000010:1.1:1788571688.337726:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355e00.
00010000:00000010:1.1:1788571688.337730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08dc00.
00010000:00000010:1.1:1788571688.337733:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355600.
00010000:00000010:1.1:1788571688.337736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08d600.
00010000:00000010:1.1:1788571688.337738:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d355b00.
00010000:00000010:1.1:1788571688.337743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b400.
00010000:00000010:1.1:1788571688.337747:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4de500.
00010000:00000010:1.1:1788571688.337751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542aa00.
00010000:00000010:1.1:1788571688.337754:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4ded00.
00010000:00000010:1.1:1788571688.337757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a600.
00010000:00000010:1.1:1788571688.337760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199000.
00010000:00000010:1.1:1788571688.337763:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a500.
00010000:00000010:1.1:1788571688.337766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ae00.
00010000:00000010:1.1:1788571688.337770:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a600.
00010000:00000010:1.1:1788571688.337773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b000.
00010000:00000010:1.1:1788571688.337776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199a00.
00010000:00000010:1.1:1788571688.337778:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a300.
00010000:00000010:1.1:1788571688.337782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ba00.
00010000:00000010:1.1:1788571688.337785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00010000:00000010:1.1:1788571688.337787:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a100.
00010000:00000010:1.1:1788571688.337791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542be00.
00010000:00000010:1.1:1788571688.337793:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545ab00.
00010000:00000010:1.1:1788571688.337796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a000.
00010000:00000010:1.1:1788571688.337799:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a800.
00010000:00000010:1.1:1788571688.337803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a200.
00010000:00000010:1.1:1788571688.337805:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a400.
00010000:00000010:1.1:1788571688.337808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542bc00.
00010000:00000010:1.1:1788571688.337813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34aa00.
00010000:00000010:1.1:1788571688.337817:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4de200.
00010000:00000010:1.1:1788571688.337820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b600.
00010000:00000010:1.1:1788571688.337823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34a600.
00010000:00000010:1.1:1788571688.337826:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4dee00.
00010000:00000010:1.1:1788571688.337829:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a400.
00010000:00000010:1.1:1788571688.337834:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:1.1:1788571688.337837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b800.
00010000:00000010:1.1:1788571688.337840:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:1.1:1788571688.337843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ac00.
00010000:00000010:1.1:1788571688.337846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ae00.
00010000:00000010:1.1:1788571688.337849:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:1.1:1788571688.337851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b200.
00010000:00000010:1.1:1788571688.337855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34b000.
00010000:00000010:1.1:1788571688.337859:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:1.1:1788571688.337862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a800.
00010000:00000010:1.1:1788571688.337865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ba00.
00010000:00000010:1.1:1788571688.337869:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:1.1:1788571688.337874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668e00.
00010000:00000010:1.1:1788571688.337877:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:1.1:1788571688.337881:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669000.
00010000:00000010:1.1:1788571688.337884:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:1.1:1788571688.337887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669a00.
00010000:00000010:1.1:1788571688.337890:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:1.1:1788571688.337893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669e00.
00010000:00000010:1.1:1788571688.337895:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4500.
00010000:00000010:1.1:1788571688.337898:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669c00.
00010000:00000010:1.1:1788571688.337901:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545aa00.
00010000:00000010:1.1:1788571688.337904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668600.
00010000:00000010:1.1:1788571688.337907:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545ad00.
00010000:00000010:1.1:1788571688.337909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668a00.
00010000:00000010:1.1:1788571688.337912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34be00.
00010000:00000010:1.1:1788571688.337915:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:1.1:1788571688.337918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669400.
00010000:00000010:1.1:1788571688.337921:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:1.1:1788571688.337924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668800.
00010000:00000010:1.1:1788571688.337927:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4d00.
00010000:00000010:1.1:1788571688.337930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669200.
00010000:00000010:1.1:1788571688.337933:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:1.1:1788571688.337936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668c00.
00010000:00000010:1.1:1788571688.337938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34a000.
00010000:00000010:1.1:1788571688.337941:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:1.1:1788571688.337944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669800.
00010000:00000010:1.1:1788571688.337947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34a200.
00010000:00000010:1.1:1788571688.337952:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155c00.
00010000:00000010:1.1:1788571688.337955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668400.
00010000:00000010:1.1:1788571688.337958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34bc00.
00010000:00000010:1.1:1788571688.337961:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155000.
00010000:00000010:1.1:1788571688.337964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669600.
00010000:00000010:1.1:1788571688.337967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34b600.
00010000:00000010:1.1:1788571688.337970:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155900.
00010000:00000010:1.1:1788571688.337974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546e400.
00010000:00000010:1.1:1788571688.337977:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34a400.
00010000:00000010:1.1:1788571688.337980:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155300.
00010000:00000010:1.1:1788571688.337983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546f800.
00010000:00000010:1.1:1788571688.337986:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34b800.
00010000:00000010:1.1:1788571688.337989:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155200.
00010000:00000010:1.1:1788571688.337992:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546ec00.
00010000:00000010:1.1:1788571688.337995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ac00.
00010000:00000010:1.1:1788571688.337997:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155b00.
00010000:00000010:1.1:1788571688.338000:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546f200.
00010000:00000010:1.1:1788571688.338003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34b200.
00010000:00000010:1.1:1788571688.338006:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155a00.
00010000:00000010:1.1:1788571688.338009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546e800.
00010000:00000010:1.1:1788571688.338012:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155e00.
00010000:00000010:1.1:1788571688.338015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546f400.
00010000:00000010:1.1:1788571688.338018:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155500.
00010000:00000010:1.1:1788571688.338022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546ea00.
00010000:00000010:1.1:1788571688.338025:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155600.
00010000:00000010:1.1:1788571688.338028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546e600.
00010000:00000010:1.1:1788571688.338031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34a800.
00010000:00000010:1.1:1788571688.338034:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155400.
00010000:00000010:1.1:1788571688.338036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546ee00.
00010000:00000010:1.1:1788571688.338040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435a00.
00010000:00000010:1.1:1788571688.338043:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2400.
00010000:00000010:1.1:1788571688.338046:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546f000.
00010000:00000010:1.1:1788571688.338049:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2000.
00010000:00000010:1.1:1788571688.338052:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546fa00.
00010000:00000010:1.1:1788571688.338054:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2e00.
00010000:00000010:1.1:1788571688.338057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546fe00.
00010000:00000010:1.1:1788571688.338060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435000.
00010000:00000010:1.1:1788571688.338063:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2200.
00010000:00000010:1.1:1788571688.338066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546e000.
00010000:00000010:1.1:1788571688.338070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434e00.
00010000:00000010:1.1:1788571688.338073:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545ac00.
00010000:00000010:1.1:1788571688.338076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546e200.
00010000:00000010:1.1:1788571688.338078:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435200.
00010000:00000010:1.1:1788571688.338081:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a000.
00010000:00000010:1.1:1788571688.338085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cba00.
00010000:00000010:1.1:1788571688.338088:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a200.
00010000:00000010:1.1:1788571688.338091:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cb000.
00010000:00000010:1.1:1788571688.338093:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a900.
00010000:00000010:1.1:1788571688.338096:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cbe00.
00010000:00000010:1.1:1788571688.338099:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434800.
00010000:00000010:1.1:1788571688.338102:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a700.
00010000:00000010:1.1:1788571688.338105:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cae00.
00010000:00000010:1.1:1788571688.338108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435400.
00010000:00000010:1.1:1788571688.338114:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000ac00.
00010000:00000010:1.1:1788571688.338143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528ca600.
00010000:00000010:1.1:1788571688.338147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434a00.
00010000:00000010:1.1:1788571688.338152:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000ab00.
00010000:00000010:1.1:1788571688.338155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528ca400.
00010000:00000010:1.1:1788571688.338169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434600.
00010000:00000010:1.1:1788571688.338172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000af00.
00010000:00000010:1.1:1788571688.338175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cb800.
00010000:00000010:1.1:1788571688.338178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000ae00.
00010000:00000010:1.1:1788571688.338181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cac00.
00010000:00000010:1.1:1788571688.338184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a800.
00010000:00000010:1.1:1788571688.338187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cb200.
00010000:00000010:1.1:1788571688.338190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a300.
00010000:00000010:1.1:1788571688.338193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528ca800.
00010000:00000010:1.1:1788571688.338196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a500.
00010000:00000010:1.1:1788571688.338199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cb400.
00010000:00000010:1.1:1788571688.338203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bff00.
00010000:00000010:1.1:1788571688.338206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528ca000.
00010000:00000010:1.1:1788571688.338211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb5600.
00010000:00000010:1.1:1788571688.338229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bfa00.
00010000:00000010:1.1:1788571688.338233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528ca200.
00010000:00000010:1.1:1788571688.338236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000ad00.
00010000:00000010:1.1:1788571688.338238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cbc00.
00010000:00000010:1.1:1788571688.338242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb4400.
00010000:00000010:1.1:1788571688.338245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a000.
00010000:00000010:1.1:1788571688.338248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953528cb600.
00010000:00000010:1.1:1788571688.338251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00010000:00000010:1.1:1788571688.338254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a100.
00010000:00000010:1.1:1788571688.338259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379628e00.
00010000:00000010:1.1:1788571688.338262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00010000:00000010:1.1:1788571688.338265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000aa00.
00010000:00000010:1.1:1788571688.338269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379629000.
00010000:00000010:1.1:1788571688.338272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99c00.
00010000:00000010:1.1:1788571688.338275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a700.
00010000:00000010:1.1:1788571688.338278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379629a00.
00010000:00000010:1.1:1788571688.338281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00010000:00000010:1.1:1788571688.338284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a900.
00010000:00000010:1.1:1788571688.338287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379629e00.
00010000:00000010:1.1:1788571688.338290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb5800.
00010000:00000010:1.1:1788571688.338293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a400.
00010000:00000010:1.1:1788571688.338296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379628000.
00010000:00000010:1.1:1788571688.338299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a200.
00010000:00000010:1.1:1788571688.338302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379628200.
00010000:00000010:1.1:1788571688.338305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536000a600.
00010000:00000010:1.1:1788571688.338308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379629c00.
00010000:00000010:1.1:1788571688.338313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284e00.
00010000:00000010:1.1:1788571688.338315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379629600.
00010000:00000010:1.1:1788571688.338318:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284f00.
00010000:00000010:1.1:1788571688.338322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379628400.
00010000:00000010:1.1:1788571688.338325:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284c00.
00010000:00000010:1.1:1788571688.338336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379629800.
00010000:00000010:1.1:1788571688.338340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284400.
00010000:00000010:1.1:1788571688.338344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379628c00.
00010000:00000010:1.1:1788571688.338347:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284a00.
00010000:00000010:1.1:1788571688.338350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379629200.
00010000:00000010:1.1:1788571688.338353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb4c00.
00010000:00000010:1.1:1788571688.338356:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284100.
00010000:00000010:1.1:1788571688.338359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379628800.
00010000:00000010:1.1:1788571688.338362:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284000.
00010000:00000010:1.1:1788571688.338365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379629400.
00010000:00000010:1.1:1788571688.338367:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284900.
00010000:00000010:1.1:1788571688.338370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb5200.
00010000:00000010:1.1:1788571688.338373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb4800.
00010000:00000010:1.1:1788571688.338376:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bfb00.
00010000:00000010:1.1:1788571688.338380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379628600.
00010000:00000010:1.1:1788571688.338388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb5400.
00010000:00000010:1.1:1788571688.338415:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf200.
00010000:00000010:1.1:1788571688.338421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971ac00.
00010000:00000010:1.1:1788571688.338424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb4a00.
00010000:00000010:1.1:1788571688.338428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bfd00.
00010000:00000010:1.1:1788571688.338431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971b200.
00010000:00000010:1.1:1788571688.338434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb4600.
00010000:00000010:1.1:1788571688.338437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bfc00.
00010000:00000010:1.1:1788571688.338440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971a800.
00010000:00000010:1.1:1788571688.338442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546f600.
00010000:00000010:1.1:1788571688.338445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf900.
00010000:00000010:1.1:1788571688.338447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971b400.
00010000:00000010:1.1:1788571688.338449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bfe00.
00010000:00000010:1.1:1788571688.338452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971aa00.
00010000:00000010:1.1:1788571688.338455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434400.
00010000:00000010:1.1:1788571688.338457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf700.
00010000:00000010:1.1:1788571688.338460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971a600.
00010000:00000010:1.1:1788571688.338462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf000.
00010000:00000010:1.1:1788571688.338465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971ae00.
00010000:00000010:1.1:1788571688.338468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb4e00.
00010000:00000010:1.1:1788571688.338471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf100.
00010000:00000010:1.1:1788571688.338474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971b000.
00010000:00000010:1.1:1788571688.338476:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf500.
00010000:00000010:1.1:1788571688.338479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971ba00.
00010000:00000010:1.1:1788571688.338481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf400.
00010000:00000010:1.1:1788571688.338484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971be00.
00010000:00000010:1.1:1788571688.338487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb5000.
00010000:00000010:1.1:1788571688.338490:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf300.
00010000:00000010:1.1:1788571688.338492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971a000.
00010000:00000010:1.1:1788571688.338495:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284200.
00010000:00000010:1.1:1788571688.338498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971a200.
00010000:00000010:1.1:1788571688.338500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284800.
00010000:00000010:1.1:1788571688.338503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971bc00.
00010000:00000010:1.1:1788571688.338506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536546fc00.
00010000:00000010:1.1:1788571688.338513:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284d00.
00010000:00000010:1.1:1788571688.338517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971b600.
00010000:00000010:1.1:1788571688.338519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08c400.
00010000:00000010:1.1:1788571688.338522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284b00.
00010000:00000010:1.1:1788571688.338525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971a400.
00010000:00000010:1.1:1788571688.338528:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284500.
00010000:00000010:1.1:1788571688.338530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537971b800.
00010000:00000010:1.1:1788571688.338532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284700.
00010000:00000010:1.1:1788571688.338537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f1e00.
00010000:00000010:1.1:1788571688.338540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284300.
00010000:00000010:1.1:1788571688.338544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f0000.
00010000:00000010:1.1:1788571688.338546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d08d800.
00010000:00000010:1.1:1788571688.338562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352284600.
00010000:00000010:1.1:1788571688.338566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f0200.
00010000:00000010:1.1:1788571688.338569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb5a00.
00010000:00000010:1.1:1788571688.338575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b000.
00010000:00000010:1.1:1788571688.338579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f1c00.
00010000:00000010:1.1:1788571688.338581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8bd00.
00010000:00000010:1.1:1788571688.338584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f1600.
00010000:00000010:1.1:1788571688.338586:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b300.
00010000:00000010:1.1:1788571688.338589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f0400.
00010000:00000010:1.1:1788571688.338592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb5e00.
00010000:00000010:1.1:1788571688.338595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8bc00.
00010000:00000010:1.1:1788571688.338597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f1800.
00010000:00000010:1.1:1788571688.338600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b100.
00010000:00000010:1.1:1788571688.338602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f0c00.
00010000:00000010:1.1:1788571688.338605:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8be00.
00010000:00000010:1.1:1788571688.338608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f1200.
00010000:00000010:1.1:1788571688.338610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b500.
00010000:00000010:1.1:1788571688.338612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f0800.
00010000:00000010:1.1:1788571688.338615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b200.
00010000:00000010:1.1:1788571688.338617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f1400.
00010000:00000010:1.1:1788571688.338619:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b600.
00010000:00000010:1.1:1788571688.338622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f0a00.
00010000:00000010:1.1:1788571688.338625:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8ba00.
00010000:00000010:1.1:1788571688.338628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f0600.
00010000:00000010:1.1:1788571688.338631:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b700.
00010000:00000010:1.1:1788571688.338633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f0e00.
00010000:00000010:1.1:1788571688.338636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb4000.
00010000:00000010:1.1:1788571688.338638:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b800.
00010000:00000010:1.1:1788571688.338641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f1000.
00010000:00000010:1.1:1788571688.338643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb4200.
00010000:00000010:1.1:1788571688.338646:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b400.
00010000:00000010:1.1:1788571688.338649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c9f1a00.
00010000:00000010:1.1:1788571688.338651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372eb5c00.
00010000:00000010:1.1:1788571688.338654:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8bb00.
00010000:00000010:1.1:1788571688.338658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb7000.
00010000:00000010:1.1:1788571688.338663:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f4a00.
00010000:00000010:1.1:1788571688.338666:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8b900.
00010000:00000010:1.1:1788571688.338669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb7a00.
00010000:00000010:1.1:1788571688.338671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00010000:00000010:1.1:1788571688.338673:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc8bf00.
00010000:00000010:1.1:1788571688.338676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb7e00.
00010000:00000010:1.1:1788571688.338678:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00010000:00000010:1.1:1788571688.338683:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0000.
00010000:00000010:1.1:1788571688.338687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb6000.
00010000:00000010:1.1:1788571688.338690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f4600.
00010000:00000010:1.1:1788571688.338692:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0600.
00010000:00000010:1.1:1788571688.338695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb6200.
00010000:00000010:1.1:1788571688.338697:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0800.
00010000:00000010:1.1:1788571688.338700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb7c00.
00010000:00000010:1.1:1788571688.338702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0b00.
00010000:00000010:1.1:1788571688.338705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb7600.
00010000:00000010:1.1:1788571688.338708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0d00.
00010000:00000010:1.1:1788571688.338710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb6e00.
00010000:00000010:1.1:1788571688.338712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f4e00.
00010000:00000010:1.1:1788571688.338715:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0700.
00010000:00000010:1.1:1788571688.338717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb6a00.
00010000:00000010:1.1:1788571688.338720:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0300.
00010000:00000010:1.1:1788571688.338722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb6800.
00010000:00000010:1.1:1788571688.338724:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0100.
00010000:00000010:1.1:1788571688.338726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb7200.
00010000:00000010:1.1:1788571688.338730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f5000.
00010000:00000010:1.1:1788571688.338733:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0400.
00010000:00000010:1.1:1788571688.338735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb6c00.
00010000:00000010:1.1:1788571688.338738:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0e00.
00010000:00000010:1.1:1788571688.338740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb7800.
00010000:00000010:1.1:1788571688.338743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f5a00.
00010000:00000010:1.1:1788571688.338745:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0c00.
00010000:00000010:1.1:1788571688.338748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb6400.
00010000:00000010:1.1:1788571688.338750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f5e00.
00010000:00000010:1.1:1788571688.338752:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0f00.
00010000:00000010:1.1:1788571688.338756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379593c00.
00010000:00000010:1.1:1788571688.338759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00010000:00000010:1.1:1788571688.338761:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0900.
00010000:00000010:1.1:1788571688.338764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379593600.
00010000:00000010:1.1:1788571688.338766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00010000:00000010:1.1:1788571688.338769:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0200.
00010000:00000010:1.1:1788571688.338772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379592400.
00010000:00000010:1.1:1788571688.338775:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b6bf800.
00010000:00000010:1.1:1788571688.338784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379593800.
00010000:00000010:1.1:1788571688.338787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f4000.
00010000:00000010:1.1:1788571688.338791:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3900.
00010000:00000010:1.1:1788571688.338794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379592c00.
00010000:00000010:1.1:1788571688.338797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f4200.
00010000:00000010:1.1:1788571688.338799:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3f00.
00010000:00000010:1.1:1788571688.338801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379593200.
00010000:00000010:1.1:1788571688.338804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f5c00.
00010000:00000010:1.1:1788571688.338807:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3e00.
00010000:00000010:1.1:1788571688.338809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379592800.
00010000:00000010:1.1:1788571688.338812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f5600.
00010000:00000010:1.1:1788571688.338814:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3a00.
00010000:00000010:1.1:1788571688.338817:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379593400.
00010000:00000010:1.1:1788571688.338823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f4400.
00010000:00000010:1.1:1788571688.338825:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3500.
00010000:00000010:1.1:1788571688.338827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379592a00.
00010000:00000010:1.1:1788571688.338830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f5800.
00010000:00000010:1.1:1788571688.338832:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3d00.
00010000:00000010:1.1:1788571688.338834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379592600.
00010000:00000010:1.1:1788571688.338837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f4c00.
00010000:00000010:1.1:1788571688.338839:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3800.
00010000:00000010:1.1:1788571688.338842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379592e00.
00010000:00000010:1.1:1788571688.338846:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7d00.
00010000:00000010:1.1:1788571688.338848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379593000.
00010000:00000010:1.1:1788571688.338850:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3100.
00010000:00000010:1.1:1788571688.338853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379593a00.
00010000:00000010:1.1:1788571688.338855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f5200.
00010000:00000010:1.1:1788571688.338857:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3400.
00010000:00000010:1.1:1788571688.338859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379593e00.
00010000:00000010:1.1:1788571688.338861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f4800.
00010000:00000010:1.1:1788571688.338864:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3000.
00010000:00000010:1.1:1788571688.338866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379592000.
00010000:00000010:1.1:1788571688.338868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d3f5400.
00010000:00000010:1.1:1788571688.338873:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374000.
00010000:00000010:1.1:1788571688.338875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379592200.
00010000:00000010:1.1:1788571688.338879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365414a00.
00010000:00000010:1.1:1788571688.338882:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374400.
00010000:00000010:1.1:1788571688.338885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352332200.
00010000:00000010:1.1:1788571688.338888:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374a00.
00010000:00000010:1.1:1788571688.338891:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352333c00.
00010000:00000010:1.1:1788571688.338894:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374e00.
00010000:00000010:1.1:1788571688.338897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352333600.
00010000:00000010:1.1:1788571688.338899:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374800.
00010000:00000010:1.1:1788571688.338901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352332400.
00010000:00000010:1.1:1788571688.338903:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374100.
00010000:00000010:1.1:1788571688.338906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352333800.
00010000:00000010:1.1:1788571688.338908:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365414600.
00010000:00000010:1.1:1788571688.338910:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374300.
00010000:00000010:1.1:1788571688.338912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352332c00.
00010000:00000010:1.1:1788571688.338915:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374d00.
00010000:00000010:1.1:1788571688.338917:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352333200.
00010000:00000010:1.1:1788571688.338919:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365414e00.
00010000:00000010:1.1:1788571688.338922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374500.
00010000:00000010:1.1:1788571688.338924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352332800.
00010000:00000010:1.1:1788571688.338928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365415000.
00010000:00000010:1.1:1788571688.338930:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374b00.
00010000:00000010:1.1:1788571688.338933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352333400.
00010000:00000010:1.1:1788571688.338936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00010000:00000010:1.1:1788571688.338938:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374f00.
00010000:00000010:1.1:1788571688.338941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352332a00.
00010000:00000010:1.1:1788571688.338943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00010000:00000010:1.1:1788571688.338945:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374200.
00010000:00000010:1.1:1788571688.338947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352332600.
00010000:00000010:1.1:1788571688.338949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365415a00.
00010000:00000010:1.1:1788571688.338953:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374900.
00010000:00000010:1.1:1788571688.338955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352332e00.
00010000:00000010:1.1:1788571688.338957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365415e00.
00010000:00000010:1.1:1788571688.338959:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374c00.
00010000:00000010:1.1:1788571688.338962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352333000.
00010000:00000010:1.1:1788571688.338964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365414000.
00010000:00000010:1.1:1788571688.338967:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7900.
00010000:00000010:1.1:1788571688.338969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352333a00.
00010000:00000010:1.1:1788571688.338972:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7b00.
00010000:00000010:1.1:1788571688.338974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352333e00.
00010000:00000010:1.1:1788571688.338976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00010000:00000010:1.1:1788571688.338984:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7700.
00010000:00000010:1.1:1788571688.338987:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352332000.
00010000:00000010:1.1:1788571688.338989:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7c00.
00010000:00000010:1.1:1788571688.338993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551ba00.
00010000:00000010:1.1:1788571688.338996:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7e00.
00010000:00000010:1.1:1788571688.338999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551be00.
00010000:00000010:1.1:1788571688.339001:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7000.
00010000:00000010:1.1:1788571688.339004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551a000.
00010000:00000010:1.1:1788571688.339007:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7f00.
00010000:00000010:1.1:1788571688.339010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551a200.
00010000:00000010:1.1:1788571688.339013:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7300.
00010000:00000010:1.1:1788571688.339015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551bc00.
00010000:00000010:1.1:1788571688.339018:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7100.
00010000:00000010:1.1:1788571688.339020:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551b600.
00010000:00000010:1.1:1788571688.339023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0b400.
00010000:00000010:1.1:1788571688.339026:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7a00.
00010000:00000010:1.1:1788571688.339028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551a400.
00010000:00000010:1.1:1788571688.339031:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7400.
00010000:00000010:1.1:1788571688.339033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551b800.
00010000:00000010:1.1:1788571688.339035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365414200.
00010000:00000010:1.1:1788571688.339038:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374700.
00010000:00000010:1.1:1788571688.339041:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551ac00.
00010000:00000010:1.1:1788571688.339043:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668000.
00010000:00000010:1.1:1788571688.339046:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895341374600.
00010000:00000010:1.1:1788571688.339049:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551b200.
00010000:00000010:1.1:1788571688.339052:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e500.
00010000:00000010:1.1:1788571688.339054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551a800.
00010000:00000010:1.1:1788571688.339057:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e000.
00010000:00000010:1.1:1788571688.339059:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551b400.
00010000:00000010:1.1:1788571688.339061:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e100.
00010000:00000010:1.1:1788571688.339064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551aa00.
00010000:00000010:1.1:1788571688.339066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668200.
00010000:00000010:1.1:1788571688.339074:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e900.
00010000:00000010:1.1:1788571688.339078:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551a600.
00010000:00000010:1.1:1788571688.339081:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365415c00.
00010000:00000010:1.1:1788571688.339083:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ed00.
00010000:00000010:1.1:1788571688.339086:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551ae00.
00010000:00000010:1.1:1788571688.339089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0aa00.
00010000:00000010:1.1:1788571688.339091:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7800.
00010000:00000010:1.1:1788571688.339094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536551b000.
00010000:00000010:1.1:1788571688.339097:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7200.
00010000:00000010:1.1:1788571688.339102:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856f200.
00010000:00000010:1.1:1788571688.339104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365415600.
00010000:00000010:1.1:1788571688.339108:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044500.
00010000:00000010:1.1:1788571688.339112:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856ec00.
00010000:00000010:1.1:1788571688.339115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365414400.
00010000:00000010:1.1:1788571688.339118:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044300.
00010000:00000010:1.1:1788571688.339120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856f600.
00010000:00000010:1.1:1788571688.339122:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044e00.
00010000:00000010:1.1:1788571688.339125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856fc00.
00010000:00000010:1.1:1788571688.339128:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044000.
00010000:00000010:1.1:1788571688.339131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856e200.
00010000:00000010:1.1:1788571688.339134:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044b00.
00010000:00000010:1.1:1788571688.339137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856e000.
00010000:00000010:1.1:1788571688.339139:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044900.
00010000:00000010:1.1:1788571688.339142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856fe00.
00010000:00000010:1.1:1788571688.339145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365415800.
00010000:00000010:1.1:1788571688.339147:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044200.
00010000:00000010:1.1:1788571688.339150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856fa00.
00010000:00000010:1.1:1788571688.339152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365414c00.
00010000:00000010:1.1:1788571688.339154:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044600.
00010000:00000010:1.1:1788571688.339157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856f000.
00010000:00000010:1.1:1788571688.339159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365415200.
00010000:00000010:1.1:1788571688.339162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044d00.
00010000:00000010:1.1:1788571688.339164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856ee00.
00010000:00000010:1.1:1788571688.339166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0a600.
00010000:00000010:1.1:1788571688.339168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044100.
00010000:00000010:1.1:1788571688.339171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856e600.
00010000:00000010:1.1:1788571688.339173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0ae00.
00010000:00000010:1.1:1788571688.339176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044800.
00010000:00000010:1.1:1788571688.339179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856e800.
00010000:00000010:1.1:1788571688.339181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365414800.
00010000:00000010:1.1:1788571688.339184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044700.
00010000:00000010:1.1:1788571688.339187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856f400.
00010000:00000010:1.1:1788571688.339189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365415400.
00010000:00000010:1.1:1788571688.339191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044f00.
00010000:00000010:1.1:1788571688.339194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534856ea00.
00010000:00000010:1.1:1788571688.339202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9d00.
00010000:00000010:1.1:1788571688.339205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e2800.
00010000:00000010:1.1:1788571688.339209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386f00.
00010000:00000010:1.1:1788571688.339213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978a400.
00010000:00000010:1.1:1788571688.339216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386d00.
00010000:00000010:1.1:1788571688.339220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978b800.
00010000:00000010:1.1:1788571688.339222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386700.
00010000:00000010:1.1:1788571688.339224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978ac00.
00010000:00000010:1.1:1788571688.339227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386400.
00010000:00000010:1.1:1788571688.339229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978b200.
00010000:00000010:1.1:1788571688.339231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386e00.
00010000:00000010:1.1:1788571688.339234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978a800.
00010000:00000010:1.1:1788571688.339236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386600.
00010000:00000010:1.1:1788571688.339239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978b400.
00010000:00000010:1.1:1788571688.339243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce3000.
00010000:00000010:1.1:1788571688.339247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a500.
00010000:00000010:1.1:1788571688.339249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978aa00.
00010000:00000010:1.1:1788571688.339252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a200.
00010000:00000010:1.1:1788571688.339254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978a600.
00010000:00000010:1.1:1788571688.339256:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a000.
00010000:00000010:1.1:1788571688.339258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978ae00.
00010000:00000010:1.1:1788571688.339261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0b000.
00010000:00000010:1.1:1788571688.339264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a700.
00010000:00000010:1.1:1788571688.339267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978b000.
00010000:00000010:1.1:1788571688.339270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a100.
00010000:00000010:1.1:1788571688.339272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978ba00.
00010000:00000010:1.1:1788571688.339274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a300.
00010000:00000010:1.1:1788571688.339277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978be00.
00010000:00000010:1.1:1788571688.339280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0ba00.
00010000:00000010:1.1:1788571688.339283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a900.
00010000:00000010:1.1:1788571688.339285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978a000.
00010000:00000010:1.1:1788571688.339287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a800.
00010000:00000010:1.1:1788571688.339290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978a200.
00010000:00000010:1.1:1788571688.339292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a400.
00010000:00000010:1.1:1788571688.339294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978bc00.
00010000:00000010:1.1:1788571688.339296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce3a00.
00010000:00000010:1.1:1788571688.339299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ae00.
00010000:00000010:1.1:1788571688.339301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537978b600.
00010000:00000010:1.1:1788571688.339303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce3e00.
00010000:00000010:1.1:1788571688.339305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a600.
00010000:00000010:1.1:1788571688.339308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0c200.
00010000:00000010:1.1:1788571688.339311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce2000.
00010000:00000010:1.1:1788571688.339313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ab00.
00010000:00000010:1.1:1788571688.339315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0dc00.
00010000:00000010:1.1:1788571688.339318:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7aa00.
00010000:00000010:1.1:1788571688.339320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0d600.
00010000:00000010:1.1:1788571688.339322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ac00.
00010000:00000010:1.1:1788571688.339324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0c400.
00010000:00000010:1.1:1788571688.339326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce2200.
00010000:00000010:1.1:1788571688.339328:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a500.
00010000:00000010:1.1:1788571688.339330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0d800.
00010000:00000010:1.1:1788571688.339332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce3c00.
00010000:00000010:1.1:1788571688.339335:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a200.
00010000:00000010:1.1:1788571688.339337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0cc00.
00010000:00000010:1.1:1788571688.339339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce3600.
00010000:00000010:1.1:1788571688.476032:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a000.
00010000:00000010:1.1:1788571688.476047:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0d200.
00010000:00000010:1.1:1788571688.476050:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a100.
00010000:00000010:1.1:1788571688.476054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0c800.
00010000:00000010:1.1:1788571688.476059:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372698200.
00010000:00000010:1.1:1788571688.476062:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a300.
00010000:00000010:1.1:1788571688.476064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0d400.
00010000:00000010:1.1:1788571688.476067:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a900.
00010000:00000010:1.1:1788571688.476069:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0ca00.
00010000:00000010:1.1:1788571688.476071:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a800.
00010000:00000010:1.1:1788571688.476093:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0c600.
00010000:00000010:1.1:1788571688.476096:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7af00.
00010000:00000010:1.1:1788571688.476099:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0ce00.
00010000:00000010:1.1:1788571688.476103:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372699a00.
00010000:00000010:1.1:1788571688.476109:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119af00.
00010000:00000010:1.1:1788571688.476111:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0d000.
00010000:00000010:1.1:1788571688.476114:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119ad00.
00010000:00000010:1.1:1788571688.476117:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0da00.
00010000:00000010:1.1:1788571688.476119:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a700.
00010000:00000010:1.1:1788571688.476122:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0de00.
00010000:00000010:1.1:1788571688.476126:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372698e00.
00010000:00000010:1.1:1788571688.476128:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a400.
00010000:00000010:1.1:1788571688.476130:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0c000.
00010000:00000010:1.1:1788571688.483932:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce2400.
00010000:00000010:1.1:1788571688.483948:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119ae00.
00010000:00000010:1.1:1788571688.483954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ec800.
00010000:00000010:1.1:1788571688.483992:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce3800.
00010000:00000010:1.1:1788571688.483997:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119a600.
00010000:00000010:1.1:1788571688.484001:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ed400.
00010000:00000010:1.1:1788571688.484013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce2c00.
00010000:00000010:1.1:1788571688.484017:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119ab00.
00010000:00000010:1.1:1788571688.484021:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516eca00.
00010000:00000010:1.1:1788571688.484024:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0be00.
00010000:00000010:1.1:1788571688.484115:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119aa00.
00010000:00000010:1.1:1788571688.484120:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ec600.
00010000:00000010:1.1:1788571688.484124:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0a000.
00010000:00000010:1.1:1788571688.484128:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535119ac00.
00010000:00000010:1.1:1788571688.484130:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ece00.
00010000:00000010:1.1:1788571688.484135:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6700.
00010000:00000010:1.1:1788571688.484138:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ed000.
00010000:00000010:1.1:1788571688.484141:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6400.
00010000:00000010:1.1:1788571688.484143:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516eda00.
00010000:00000010:1.1:1788571688.484145:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6e00.
00010000:00000010:1.1:1788571688.484149:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ede00.
00010000:00000010:1.1:1788571688.484151:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0a200.
00010000:00000010:1.1:1788571688.484153:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6600.
00010000:00000010:1.1:1788571688.484156:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ec000.
00010000:00000010:1.1:1788571688.484158:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6b00.
00010000:00000010:1.1:1788571688.484160:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ec200.
00010000:00000010:1.1:1788571688.484163:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6a00.
00010000:00000010:1.1:1788571688.484165:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516edc00.
00010000:00000010:1.1:1788571688.484167:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce3200.
00010000:00000010:1.1:1788571688.484170:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6c00.
00010000:00000010:1.1:1788571688.484173:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ed600.
00010000:00000010:1.1:1788571688.484176:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce2800.
00010000:00000010:1.1:1788571688.484179:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ad00.
00010000:00000010:1.1:1788571688.484181:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ec400.
00010000:00000010:1.1:1788571688.484187:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b700.
00010000:00000010:1.1:1788571688.484190:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ed800.
00010000:00000010:1.1:1788571688.484193:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce3400.
00010000:00000010:1.1:1788571688.484195:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b400.
00010000:00000010:1.1:1788571688.484198:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ecc00.
00010000:00000010:1.1:1788571688.484200:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0b600.
00010000:00000010:1.1:1788571688.484204:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0be00.
00010000:00000010:1.1:1788571688.484206:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ed200.
00010000:00000010:1.1:1788571688.484209:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0bc00.
00010000:00000010:1.1:1788571688.484211:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b600.
00010000:00000010:1.1:1788571688.484216:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383ee00.
00010000:00000010:1.1:1788571688.484219:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0bb00.
00010000:00000010:1.1:1788571688.484223:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f000.
00010000:00000010:1.1:1788571688.484226:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0ba00.
00010000:00000010:1.1:1788571688.484229:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383fa00.
00010000:00000010:1.1:1788571688.484231:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce2a00.
00010000:00000010:1.1:1788571688.484235:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0bc00.
00010000:00000010:1.1:1788571688.484238:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383fe00.
00010000:00000010:1.1:1788571688.484241:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb7400.
00010000:00000010:1.1:1788571688.484243:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b500.
00010000:00000010:1.1:1788571688.484246:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e000.
00010000:00000010:1.1:1788571688.484249:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372fb6600.
00010000:00000010:1.1:1788571688.484258:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b200.
00010000:00000010:1.1:1788571688.484262:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e200.
00010000:00000010:1.1:1788571688.484264:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b000.
00010000:00000010:1.1:1788571688.484267:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383fc00.
00010000:00000010:1.1:1788571688.484271:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b100.
00010000:00000010:1.1:1788571688.484274:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f600.
00010000:00000010:1.1:1788571688.484277:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b300.
00010000:00000010:1.1:1788571688.484279:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e400.
00010000:00000010:1.1:1788571688.484281:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b900.
00010000:00000010:1.1:1788571688.484284:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f800.
00010000:00000010:1.1:1788571688.484286:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0b800.
00010000:00000010:1.1:1788571688.484288:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383ec00.
00010000:00000010:1.1:1788571688.484291:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0bf00.
00010000:00000010:1.1:1788571688.484294:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f200.
00010000:00000010:1.1:1788571688.484296:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895380f0bd00.
00010000:00000010:1.1:1788571688.484299:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e800.
00010000:00000010:1.1:1788571688.484305:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9900.
00010000:00000010:1.1:1788571688.484308:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f400.
00010000:00000010:1.1:1788571688.484311:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce2600.
00010000:00000010:1.1:1788571688.484313:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9800.
00010000:00000010:1.1:1788571688.484316:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383ea00.
00010000:00000010:1.1:1788571688.484319:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895342ce2e00.
00010000:00000010:1.1:1788571688.484340:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9f00.
00010000:00000010:1.1:1788571688.484344:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e600.
00010000:00000010:1.1:1788571688.484350:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e3400.
00010000:00000010:1.1:1788571688.484355:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8600.
00010000:00000010:1.1:1788571688.484360:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365457400.
00010000:00000010:1.1:1788571688.484365:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e2a00.
00010000:00000010:1.1:1788571688.484368:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8900.
00010000:00000010:1.1:1788571688.484371:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365456a00.
00010000:00000010:1.1:1788571688.484374:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e2600.
00010000:00000010:1.1:1788571688.484377:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9700.
00010000:00000010:1.1:1788571688.484380:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365456c00.
00010000:00000010:1.1:1788571688.484382:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e2e00.
00010000:00000010:1.1:1788571688.484386:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9400.
00010000:00000010:1.1:1788571688.484389:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365457600.
00010000:00000010:1.1:1788571688.484427:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0a400.
00010000:00000010:1.1:1788571688.484432:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9e00.
00010000:00000010:1.1:1788571688.484435:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365457e00.
00010000:00000010:1.1:1788571688.484438:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0b800.
00010000:00000010:1.1:1788571688.484440:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9600.
00010000:00000010:1.1:1788571688.484443:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365456000.
00010000:00000010:1.1:1788571688.484446:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9b00.
00010000:00000010:1.1:1788571688.484450:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e16400.
00010000:00000010:1.1:1788571688.484463:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e3000.
00010000:00000010:1.1:1788571688.484467:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9a00.
00010000:00000010:1.1:1788571688.484470:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e16c00.
00010000:00000010:1.1:1788571688.484472:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e3a00.
00010000:00000010:1.1:1788571688.484475:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9c00.
00010000:00000010:1.1:1788571688.484478:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e16800.
00010000:00000010:1.1:1788571688.484480:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e3e00.
00010000:00000010:1.1:1788571688.484482:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9500.
00010000:00000010:1.1:1788571688.484485:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e16a00.
00010000:00000010:1.1:1788571688.484488:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e2000.
00010000:00000010:1.1:1788571688.484493:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9200.
00010000:00000010:1.1:1788571688.484496:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e16e00.
00010000:00000010:1.1:1788571688.484498:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e2200.
00010000:00000010:1.1:1788571688.484502:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9000.
00010000:00000010:1.1:1788571688.484506:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e17a00.
00010000:00000010:1.1:1788571688.484510:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372699800.
00010000:00000010:1.1:1788571688.484513:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9100.
00010000:00000010:1.1:1788571688.484515:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e16000.
00010000:00000010:1.1:1788571688.484519:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372698c00.
00010000:00000010:1.1:1788571688.484521:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953726f9300.
00010000:00000010:1.1:1788571688.484525:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e17c00.
00010000:00000010:1.1:1788571688.484530:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48400.
00010000:00000010:1.1:1788571688.484537:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2b00.
00010000:00000010:1.1:1788571688.484540:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e17e00.
00010000:00000010:1.1:1788571688.484542:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2a00.
00010000:00000010:1.1:1788571688.484545:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e16600.
00010000:00000010:1.1:1788571688.484548:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2c00.
00010000:00000010:1.1:1788571688.484561:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e17200.
00010000:00000010:1.1:1788571688.484564:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2500.
00010000:00000010:1.1:1788571688.484567:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e17600.
00010000:00000010:1.1:1788571688.484571:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48000.
00010000:00000010:1.1:1788571688.484573:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2200.
00010000:00000010:1.1:1788571688.484576:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e16200.
00010000:00000010:1.1:1788571688.484579:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2000.
00010000:00000010:1.1:1788571688.484581:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e17000.
00010000:00000010:1.1:1788571688.484988:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2100.
00010000:00000010:1.1:1788571688.484991:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e17400.
00010000:00000010:1.1:1788571688.484993:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2300.
00010000:00000010:1.1:1788571688.484996:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372e17800.
00010000:00000010:1.1:1788571688.484998:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2900.
00010000:00000010:1.1:1788571688.485015:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d194000.
00010000:00000010:1.1:1788571688.485019:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2800.
00010000:00000010:1.1:1788571688.485214:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d195c00.
00010000:00000010:1.1:1788571688.485218:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e3c00.
00010000:00000010:1.1:1788571688.485221:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2f00.
00010000:00000010:1.1:1788571688.485223:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d194c00.
00010000:00000010:1.1:1788571688.485226:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372698600.
00010000:00000010:1.1:1788571688.485228:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2d00.
00010000:00000010:1.1:1788571688.485230:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d194a00.
00010000:00000010:1.1:1788571688.485233:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372698a00.
00010000:00000010:1.1:1788571688.485238:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2700.
00010000:00000010:1.1:1788571688.485241:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d194800.
00010000:00000010:1.1:1788571688.485243:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2400.
00010000:00000010:1.1:1788571688.485245:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d195a00.
00010000:00000010:1.1:1788571688.485247:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2e00.
00010000:00000010:1.1:1788571688.485250:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d194400.
00010000:00000010:1.1:1788571688.485252:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e2600.
00010000:00000010:1.1:1788571688.485254:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d195e00.
00010000:00000010:1.1:1788571688.485257:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372699600.
00010000:00000010:1.1:1788571688.485262:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5200.
00010000:00000010:1.1:1788571688.485265:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d194600.
00010000:00000010:1.1:1788571688.485268:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372698800.
00010000:00000010:1.1:1788571688.485270:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5e00.
00010000:00000010:1.1:1788571688.485273:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d195200.
00010000:00000010:1.1:1788571688.485275:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5700.
00010000:00000010:1.1:1788571688.485278:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d195600.
00010000:00000010:1.1:1788571688.485280:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5500.
00010000:00000010:1.1:1788571688.485282:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d194e00.
00010000:00000010:1.1:1788571688.485284:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e3600.
00010000:00000010:1.1:1788571688.485287:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5d00.
00010000:00000010:1.1:1788571688.485289:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d195000.
00010000:00000010:1.1:1788571688.485292:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5f00.
00010000:00000010:1.1:1788571688.485294:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d195400.
00010000:00000010:1.1:1788571688.485297:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5600.
00010000:00000010:1.1:1788571688.485300:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d195800.
00010000:00000010:1.1:1788571688.485303:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0ac00.
00010000:00000010:1.1:1788571688.485305:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5000.
00010000:00000010:1.1:1788571688.485308:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d194200.
00010000:00000010:1.1:1788571688.485310:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5c00.
00010000:00000010:1.1:1788571688.485314:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc19200.
00010000:00000010:1.1:1788571688.485318:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5900.
00010000:00000010:1.1:1788571688.485320:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc19400.
00010000:00000010:1.1:1788571688.485324:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e2400.
00010000:00000010:1.1:1788571688.485326:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5a00.
00010000:00000010:1.1:1788571688.485330:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc18600.
00010000:00000010:1.1:1788571688.485332:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e3800.
00010000:00000010:1.1:1788571688.485335:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5100.
00010000:00000010:1.1:1788571688.485338:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc19a00.
00010000:00000010:1.1:1788571688.485341:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48800.
00010000:00000010:1.1:1788571688.485343:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5b00.
00010000:00000010:1.1:1788571688.485347:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc18000.
00010000:00000010:1.1:1788571688.485351:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49400.
00010000:00000010:1.1:1788571688.485354:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5800.
00010000:00000010:1.1:1788571688.485356:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc19c00.
00010000:00000010:1.1:1788571688.485359:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49200.
00010000:00000010:1.1:1788571688.485361:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5400.
00010000:00000010:1.1:1788571688.485364:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc18400.
00010000:00000010:1.1:1788571688.485366:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49c00.
00010000:00000010:1.1:1788571688.485368:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89537c9f5300.
00010000:00000010:1.1:1788571688.485372:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee7000.
00010000:00000010:1.1:1788571688.485377:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535124be00.
00010000:00000010:1.1:1788571688.485580:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee7a00.
00010000:00000010:1.1:1788571688.485595:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535124b100.
00010000:00000010:1.1:1788571688.485598:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee7e00.
00010000:00000010:1.1:1788571688.485602:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535124b000.
00010000:00000010:1.1:1788571688.485607:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee6000.
00010000:00000010:1.1:1788571688.485610:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535124ba00.
00010000:00000010:1.1:1788571688.485613:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee6200.
00010000:00000010:1.1:1788571688.485615:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535124b600.
00010000:00000010:1.1:1788571688.485618:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee7c00.
00010000:00000010:1.1:1788571688.485624:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535124b500.
00010000:00000010:1.1:1788571688.485627:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee7600.
00010000:00000010:1.1:1788571688.485629:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535124b400.
00010000:00000010:1.1:1788571688.485634:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41a200.
00010000:00000010:1.1:1788571688.485637:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89535124bb00.
00010000:00000010:1.1:1788571688.485641:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41bc00.
00010000:00000010:1.1:1788571688.485645:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a4b7500.
00010000:00000010:1.1:1788571688.485648:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41b600.
00010000:00000010:1.1:1788571688.485650:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0b200.
00010000:00000010:1.1:1788571688.485654:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a4b7700.
00010000:00000010:1.1:1788571688.485657:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41a400.
00010000:00000010:1.1:1788571688.485660:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e2c00.
00010000:00000010:1.1:1788571688.485663:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a4b7a00.
00010000:00000010:1.1:1788571688.485666:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41b800.
00010000:00000010:1.1:1788571688.485668:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8e3200.
00010000:00000010:1.1:1788571688.485677:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a4b7d00.
00010000:00000010:1.1:1788571688.485681:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41ac00.
00010000:00000010:1.1:1788571688.485684:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372699000.
00010000:00000010:1.1:1788571688.485687:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a4b7200.
00010000:00000010:1.1:1788571688.485690:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41b200.
00010000:00000010:1.1:1788571688.485694:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a4b7c00.
00010000:00000010:1.1:1788571688.485697:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41a800.
00010000:00000010:1.1:1788571688.485699:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a4b7b00.
00010000:00000010:1.1:1788571688.485702:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41b400.
00010000:00000010:1.1:1788571688.485705:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0a800.
00010000:00000010:1.1:1788571688.485724:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a4b7e00.
00010000:00000010:1.1:1788571688.485728:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41aa00.
00010000:00000010:1.1:1788571688.485733:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee6400.
00010000:00000010:1.1:1788571688.485735:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a4b7000.
00010000:00000010:1.1:1788571688.485738:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41a600.
00010000:00000010:1.1:1788571688.485742:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee7800.
00010000:00000010:1.1:1788571688.485746:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8d00.
00010000:00000010:1.1:1788571688.485749:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41ae00.
00010000:00000010:1.1:1788571688.485752:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8800.
00010000:00000010:1.1:1788571688.485755:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41b000.
00010000:00000010:1.1:1788571688.485758:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee6c00.
00010000:00000010:1.1:1788571688.485761:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8100.
00010000:00000010:1.1:1788571688.485764:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41ba00.
00010000:00000010:1.1:1788571688.485770:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560700.
00010000:00000010:1.1:1788571688.485774:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352356c00.
00010000:00000010:1.1:1788571688.485778:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560200.
00010000:00000010:1.1:1788571688.485782:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352357200.
00010000:00000010:1.1:1788571688.485786:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c6600.
00010000:00000010:1.1:1788571688.485789:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560800.
00010000:00000010:1.1:1788571688.485792:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352356800.
00010000:00000010:1.1:1788571688.485795:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560000.
00010000:00000010:1.1:1788571688.485799:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352357400.
00010000:00000010:1.1:1788571688.485801:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560f00.
00010000:00000010:1.1:1788571688.485804:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352356a00.
00010000:00000010:1.1:1788571688.485808:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c7800.
00010000:00000010:1.1:1788571688.485823:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560300.
00010000:00000010:1.1:1788571688.485827:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352356600.
00010000:00000010:1.1:1788571688.485829:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c6200.
00010000:00000010:1.1:1788571688.485832:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560c00.
00010000:00000010:1.1:1788571688.485835:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352356e00.
00010000:00000010:1.1:1788571688.485838:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560600.
00010000:00000010:1.1:1788571688.485840:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352357000.
00010000:00000010:1.1:1788571688.485844:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560e00.
00010000:00000010:1.1:1788571688.485847:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352357a00.
00010000:00000010:1.1:1788571688.485853:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1200.
00010000:00000010:1.1:1788571688.485857:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352357e00.
00010000:00000010:1.1:1788571688.485860:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c6800.
00010000:00000010:1.1:1788571688.485863:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1600.
00010000:00000010:1.1:1788571688.485865:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352356000.
00010000:00000010:1.1:1788571688.485868:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee7200.
00010000:00000010:1.1:1788571688.485871:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1f00.
00010000:00000010:1.1:1788571688.485874:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352356200.
00010000:00000010:1.1:1788571688.485876:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1400.
00010000:00000010:1.1:1788571688.485879:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352357c00.
00010000:00000010:1.1:1788571688.485882:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1800.
00010000:00000010:1.1:1788571688.485888:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5600.
00010000:00000010:1.1:1788571688.485891:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1c00.
00010000:00000010:1.1:1788571688.485896:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4400.
00010000:00000010:1.1:1788571688.485899:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee6800.
00010000:00000010:1.1:1788571688.485901:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1700.
00010000:00000010:1.1:1788571688.485904:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5800.
00010000:00000010:1.1:1788571688.485908:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee7400.
00010000:00000010:1.1:1788571688.485910:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1d00.
00010000:00000010:1.1:1788571688.485913:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4c00.
00010000:00000010:1.1:1788571688.485918:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb000.
00010000:00000010:1.1:1788571688.485922:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5200.
00010000:00000010:1.1:1788571688.485925:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb300.
00010000:00000010:1.1:1788571688.485927:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4800.
00010000:00000010:1.1:1788571688.485930:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c6000.
00010000:00000010:1.1:1788571688.485932:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb500.
00010000:00000010:1.1:1788571688.485935:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5400.
00010000:00000010:1.1:1788571688.485938:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edbd00.
00010000:00000010:1.1:1788571688.485940:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4a00.
00010000:00000010:1.1:1788571688.485943:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c7400.
00010000:00000010:1.1:1788571688.485946:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edbe00.
00010000:00000010:1.1:1788571688.485949:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4600.
00010000:00000010:1.1:1788571688.485952:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c6400.
00010000:00000010:1.1:1788571688.485955:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb200.
00010000:00000010:1.1:1788571688.485958:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4e00.
00010000:00000010:1.1:1788571688.485960:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c7c00.
00010000:00000010:1.1:1788571688.485962:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edba00.
00010000:00000010:1.1:1788571688.485965:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5000.
00010000:00000010:1.1:1788571688.485967:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c6c00.
00010000:00000010:1.1:1788571688.485970:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb800.
00010000:00000010:1.1:1788571688.485973:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5a00.
00010000:00000010:1.1:1788571688.485977:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c7000.
00010000:00000010:1.1:1788571688.485979:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb400.
00010000:00000010:1.1:1788571688.485982:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5e00.
00010000:00000010:1.1:1788571688.485984:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c6a00.
00010000:00000010:1.1:1788571688.485986:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb600.
00010000:00000010:1.1:1788571688.485989:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4000.
00010000:00000010:1.1:1788571688.485991:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c6e00.
00010000:00000010:1.1:1788571688.485994:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb700.
00010000:00000010:1.1:1788571688.485997:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4200.
00010000:00000010:1.1:1788571688.485999:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb900.
00010000:00000010:1.1:1788571688.486003:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5c00.
00010000:00000010:1.1:1788571688.486005:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edbc00.
00010000:00000010:1.1:1788571688.486008:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341518800.
00010000:00000010:1.1:1788571688.486012:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb100.
00010000:00000010:1.1:1788571688.486016:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341519000.
00010000:00000010:1.1:1788571688.486020:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c7200.
00010000:00000010:1.1:1788571688.486022:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edbf00.
00010000:00000010:1.1:1788571688.486025:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341518a00.
00010000:00000010:1.1:1788571688.486030:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461300.
00010000:00000010:1.1:1788571688.486033:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341518c00.
00010000:00000010:1.1:1788571688.486036:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461f00.
00010000:00000010:1.1:1788571688.486039:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341519400.
00010000:00000010:1.1:1788571688.486042:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461400.
00010000:00000010:1.1:1788571688.486045:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341518e00.
00010000:00000010:1.1:1788571688.486047:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461d00.
00010000:00000010:1.1:1788571688.486050:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341519200.
00010000:00000010:1.1:1788571688.486052:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461900.
00010000:00000010:1.1:1788571688.486056:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365708e00.
00010000:00000010:1.1:1788571688.486058:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461600.
00010000:00000010:1.1:1788571688.486062:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365709000.
00010000:00000010:1.1:1788571688.486066:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c7e00.
00010000:00000010:1.1:1788571688.486068:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461c00.
00010000:00000010:1.1:1788571688.486071:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365708600.
00010000:00000010:1.1:1788571688.486073:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c7a00.
00010000:00000010:1.1:1788571688.486078:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0edb00.
00010000:00000010:1.1:1788571688.486081:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365708a00.
00010000:00000010:1.1:1788571688.486083:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953720c7600.
00010000:00000010:1.1:1788571688.486086:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0eda00.
00010000:00000010:1.1:1788571688.486089:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365709400.
00010000:00000010:1.1:1788571688.486092:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0ed600.
00010000:00000010:1.1:1788571688.486095:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365708800.
00010000:00000010:1.1:1788571688.486098:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0edf00.
00010000:00000010:1.1:1788571688.486102:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365709200.
00010000:00000010:1.1:1788571688.486104:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee6a00.
00010000:00000010:1.1:1788571688.486107:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0ed400.
00010000:00000010:1.1:1788571688.486110:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365708c00.
00010000:00000010:1.1:1788571688.486113:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee6600.
00010000:00000010:1.1:1788571688.486116:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0ede00.
00010000:00000010:1.1:1788571688.486119:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365709800.
00010000:00000010:1.1:1788571688.486122:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365456e00.
00010000:00000010:1.1:1788571688.486125:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0ed500.
00010000:00000010:1.1:1788571688.486128:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365708400.
00010000:00000010:1.1:1788571688.486130:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365456800.
00010000:00000010:1.1:1788571688.486133:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0ed000.
00010000:00000010:1.1:1788571688.486136:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365709600.
00010000:00000010:1.1:1788571688.486139:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365456600.
00010000:00000010:1.1:1788571688.486142:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0edc00.
00010000:00000010:1.1:1788571688.486145:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365709c00.
00010000:00000010:1.1:1788571688.486148:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0ed100.
00010000:00000010:1.1:1788571688.486151:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365708200.
00010000:00000010:1.1:1788571688.486155:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0ed300.
00010000:00000010:1.1:1788571688.486158:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365708000.
00010000:00000010:1.1:1788571688.486160:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365456200.
00010000:00000010:1.1:1788571688.486163:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0ed800.
00010000:00000010:1.1:1788571688.486165:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365709e00.
00010000:00000010:1.1:1788571688.486168:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0edd00.
00010000:00000010:1.1:1788571688.486171:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365709a00.
00010000:00000010:1.1:1788571688.486174:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6500.
00010000:00000010:1.1:1788571688.486178:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166de00.
00010000:00000010:1.1:1788571688.486209:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6200.
00010000:00000010:1.1:1788571688.486214:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166da00.
00010000:00000010:1.1:1788571688.486217:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6000.
00010000:00000010:1.1:1788571688.486221:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166dc00.
00010000:00000010:1.1:1788571688.486224:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365457800.
00010000:00000010:1.1:1788571688.486227:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6100.
00010000:00000010:1.1:1788571688.486230:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166d400.
00010000:00000010:1.1:1788571688.486233:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365457a00.
00010000:00000010:1.1:1788571688.486236:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8700.
00010000:00000010:1.1:1788571688.486239:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166c000.
00010000:00000010:1.1:1788571688.486242:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365457c00.
00010000:00000010:1.1:1788571688.486246:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8f00.
00010000:00000010:1.1:1788571688.486249:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166d000.
00010000:00000010:1.1:1788571688.486251:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365457200.
00010000:00000010:1.1:1788571688.486254:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8e00.
00010000:00000010:1.1:1788571688.486256:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166ca00.
00010000:00000010:1.1:1788571688.486259:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365456400.
00010000:00000010:1.1:1788571688.486261:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8000.
00010000:00000010:1.1:1788571688.486264:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166ce00.
00010000:00000010:1.1:1788571688.486266:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cee6e00.
00010000:00000010:1.1:1788571688.486273:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8300.
00010000:00000010:1.1:1788571688.486277:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166c200.
00010000:00000010:1.1:1788571688.486280:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365457000.
00010000:00000010:1.1:1788571688.486285:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8500.
00010000:00000010:1.1:1788571688.486288:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166c600.
00010000:00000010:1.1:1788571688.486291:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc19800.
00010000:00000010:1.1:1788571688.486294:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1e00.
00010000:00000010:1.1:1788571688.486297:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534166d600.
00010000:00000010:1.1:1788571688.486299:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1900.
00010000:00000010:1.1:1788571688.486303:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00010000:00000010:1.1:1788571688.486306:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6300.
00010000:00000010:1.1:1788571688.486311:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00010000:00000010:1.1:1788571688.486313:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6900.
00010000:00000010:1.1:1788571688.486317:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00010000:00000010:1.1:1788571688.486319:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6800.
00010000:00000010:1.1:1788571688.486322:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00010000:00000010:1.1:1788571688.486324:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6f00.
00010000:00000010:1.1:1788571688.486327:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00010000:00000010:1.1:1788571688.486329:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b2b6d00.
00010000:00000010:1.1:1788571688.486338:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00010000:00000010:1.1:1788571688.486341:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc19600.
00010000:00000010:1.1:1788571688.486346:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7a700.
00010000:00000010:1.1:1788571688.486349:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00010000:00000010:1.1:1788571688.486352:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7ab00.
00010000:00000010:1.1:1788571688.486354:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00010000:00000010:1.1:1788571688.486357:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7ae00.
00010000:00000010:1.1:1788571688.486359:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00010000:00000010:1.1:1788571688.486362:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7a000.
00010000:00000010:1.1:1788571688.486364:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:1.1:1788571688.486367:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7ac00.
00010000:00000010:1.1:1788571688.486370:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00010000:00000010:1.1:1788571688.486373:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7a900.
00010000:00000010:1.1:1788571688.486376:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00010000:00000010:1.1:1788571688.486379:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7a100.
00010000:00000010:1.1:1788571688.486381:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00010000:00000010:1.1:1788571688.486384:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1500.
00010000:00000010:1.1:1788571688.486387:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00010000:00000010:1.1:1788571688.486389:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1100.
00010000:00000010:1.1:1788571688.487184:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00010000:00000010:1.1:1788571688.487188:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41be00.
00010000:00000010:1.1:1788571688.487191:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1300.
00010000:00000010:1.1:1788571688.487195:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00010000:00000010:1.1:1788571688.487197:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1000.
00010000:00000010:1.1:1788571688.487202:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947d600.
00010000:00000010:1.1:1788571688.487206:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1b00.
00010000:00000010:1.1:1788571688.487210:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947da00.
00010000:00000010:1.1:1788571688.487213:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc19e00.
00010000:00000010:1.1:1788571688.487216:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372ef1a00.
00010000:00000010:1.1:1788571688.487219:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947d200.
00010000:00000010:1.1:1788571688.487222:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc18e00.
00010000:00000010:1.1:1788571688.487225:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7ad00.
00010000:00000010:1.1:1788571688.487228:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947d400.
00010000:00000010:1.1:1788571688.487231:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc18800.
00010000:00000010:1.1:1788571688.487233:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7aa00.
00010000:00000010:1.1:1788571688.487237:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947ce00.
00010000:00000010:1.1:1788571688.487239:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc18200.
00010000:00000010:1.1:1788571688.487241:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7a400.
00010000:00000010:1.1:1788571688.487244:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947dc00.
00010000:00000010:1.1:1788571688.487246:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc19000.
00010000:00000010:1.1:1788571688.487248:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7a500.
00010000:00000010:1.1:1788571688.487250:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947c200.
00010000:00000010:1.1:1788571688.487253:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc18a00.
00010000:00000010:1.1:1788571688.487256:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7a300.
00010000:00000010:1.1:1788571688.487259:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947c400.
00010000:00000010:1.1:1788571688.487261:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc18c00.
00010000:00000010:1.1:1788571688.487274:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177000.
00010000:00000010:1.1:1788571688.487278:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947ca00.
00010000:00000010:1.1:1788571688.487281:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177500.
00010000:00000010:1.1:1788571688.487283:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947d000.
00010000:00000010:1.1:1788571688.487285:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177a00.
00010000:00000010:1.1:1788571688.487287:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947c600.
00010000:00000010:1.1:1788571688.487290:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177200.
00010000:00000010:1.1:1788571688.487293:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947cc00.
00010000:00000010:1.1:1788571688.487296:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66e00.
00010000:00000010:1.1:1788571688.487299:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177c00.
00010000:00000010:1.1:1788571688.487303:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162e00.
00010000:00000010:1.1:1788571688.487306:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66a00.
00010000:00000010:1.1:1788571688.487309:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177300.
00010000:00000010:1.1:1788571688.487312:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163600.
00010000:00000010:1.1:1788571688.487315:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177600.
00010000:00000010:1.1:1788571688.487317:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163200.
00010000:00000010:1.1:1788571688.487321:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177100.
00010000:00000010:1.1:1788571688.490761:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162a00.
00010000:00000010:1.1:1788571688.490773:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177800.
00010000:00000010:1.1:1788571688.490777:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163c00.
00010000:00000010:1.1:1788571688.490783:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67c00.
00010000:00000010:1.1:1788571688.490785:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177f00.
00010000:00000010:1.1:1788571688.490788:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162600.
00010000:00000010:1.1:1788571688.490791:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66c00.
00010000:00000010:1.1:1788571688.490794:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895360177b00.
00010000:00000010:1.1:1788571688.490797:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162800.
00010000:00000010:1.1:1788571688.490809:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67000.
00010000:00000010:1.1:1788571688.490825:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb500.
00010000:00000010:1.1:1788571688.490830:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163400.
00010000:00000010:1.1:1788571688.490835:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb000.
00010000:00000010:1.1:1788571688.490837:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163800.
00010000:00000010:1.1:1788571688.490840:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66200.
00010000:00000010:1.1:1788571688.490843:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb800.
00010000:00000010:1.1:1788571688.490845:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162200.
00010000:00000010:1.1:1788571688.490848:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89534b41a000.
00010000:00000010:1.1:1788571688.490866:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb300.
00010000:00000010:1.1:1788571688.490872:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030200.
00010000:00000010:1.1:1788571688.490876:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352357600.
00010000:00000010:1.1:1788571688.490879:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbbd00.
00010000:00000010:1.1:1788571688.490882:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030800.
00010000:00000010:1.1:1788571688.490884:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352356400.
00010000:00000010:1.1:1788571688.490887:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb100.
00010000:00000010:1.1:1788571688.490889:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030a00.
00010000:00000010:1.1:1788571688.490891:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb200.
00010000:00000010:1.1:1788571688.490895:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031a00.
00010000:00000010:1.1:1788571688.490899:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48c00.
00010000:00000010:1.1:1788571688.490902:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb600.
00010000:00000010:1.1:1788571688.490905:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031000.
00010000:00000010:1.1:1788571688.490907:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbbb00.
00010000:00000010:1.1:1788571688.490910:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030e00.
00010000:00000010:1.1:1788571688.490912:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb900.
00010000:00000010:1.1:1788571688.490915:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031e00.
00010000:00000010:1.1:1788571688.490918:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbbc00.
00010000:00000010:1.1:1788571688.490920:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030000.
00010000:00000010:1.1:1788571688.490923:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbbf00.
00010000:00000010:1.1:1788571688.490925:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031400.
00010000:00000010:1.1:1788571688.490931:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560a00.
00010000:00000010:1.1:1788571688.490933:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030600.
00010000:00000010:1.1:1788571688.490936:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560500.
00010000:00000010:1.1:1788571688.490938:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031200.
00010000:00000010:1.1:1788571688.490942:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372699400.
00010000:00000010:1.1:1788571688.490945:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560900.
00010000:00000010:1.1:1788571688.490949:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f4c00.
00010000:00000010:1.1:1788571688.490954:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372698000.
00010000:00000010:1.1:1788571688.490956:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560400.
00010000:00000010:1.1:1788571688.490959:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f4600.
00010000:00000010:1.1:1788571688.490962:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560100.
00010000:00000010:1.1:1788571688.490965:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372698400.
00010000:00000010:1.1:1788571688.490967:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345560b00.
00010000:00000010:1.1:1788571688.490970:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372699c00.
00010000:00000010:1.1:1788571688.490972:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895352357800.
00010000:00000010:1.1:1788571688.490982:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461e00.
00010000:00000010:1.1:1788571688.490987:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f5800.
00010000:00000010:1.1:1788571688.490990:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66000.
00010000:00000010:1.1:1788571688.490993:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461500.
00010000:00000010:1.1:1788571688.490996:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f5e00.
00010000:00000010:1.1:1788571688.490999:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461200.
00010000:00000010:1.1:1788571688.491002:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f5400.
00010000:00000010:1.1:1788571688.491004:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461100.
00010000:00000010:1.1:1788571688.495171:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f5c00.
00010000:00000010:1.1:1788571688.495192:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461b00.
00010000:00000010:1.1:1788571688.495196:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f5000.
00010000:00000010:1.1:1788571688.495199:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461800.
00010000:00000010:1.1:1788571688.495204:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f4a00.
00010000:00000010:1.1:1788571688.495208:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67200.
00010000:00000010:1.1:1788571688.495211:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461700.
00010000:00000010:1.1:1788571688.495225:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f5600.
00010000:00000010:1.1:1788571688.495228:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66600.
00010000:00000010:1.1:1788571688.495231:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461000.
00010000:00000010:1.1:1788571688.495233:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f4000.
00010000:00000010:1.1:1788571688.495236:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348461a00.
00010000:00000010:1.1:1788571688.495238:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f4800.
00010000:00000010:1.1:1788571688.495242:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e400.
00010000:00000010:1.1:1788571688.495248:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49800.
00010000:00000010:1.1:1788571688.495250:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67800.
00010000:00000010:1.1:1788571688.495253:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ea00.
00010000:00000010:1.1:1788571688.495256:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49e00.
00010000:00000010:1.1:1788571688.495259:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66800.
00010000:00000010:1.1:1788571688.495261:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e600.
00010000:00000010:1.1:1788571688.495265:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48200.
00010000:00000010:1.1:1788571688.495267:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67a00.
00010000:00000010:1.1:1788571688.495269:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ec00.
00010000:00000010:1.1:1788571688.495272:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49600.
00010000:00000010:1.1:1788571688.495274:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ee00.
00010000:00000010:1.1:1788571688.495276:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00010000:00000010:1.1:1788571688.495278:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e300.
00010000:00000010:1.1:1788571688.495281:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49000.
00010000:00000010:1.1:1788571688.495284:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e200.
00010000:00000010:1.1:1788571688.495286:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49a00.
00010000:00000010:1.1:1788571688.495288:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67e00.
00010000:00000010:1.1:1788571688.495291:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ef00.
00010000:00000010:1.1:1788571688.495293:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48a00.
00010000:00000010:1.1:1788571688.495296:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372699e00.
00010000:00000010:1.1:1788571688.495313:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b0ed200.
00010000:00000010:1.1:1788571688.495316:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67600.
00010000:00000010:1.1:1788571688.495319:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f5200.
00010000:00000010:1.1:1788571688.495335:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f5a00.
00010000:00000010:1.1:1788571688.495338:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895372044400.
00010000:00000010:1.1:1788571688.495348:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89537c8f4e00.
00010000:00000010:1.1:1788571695.984638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:1.1:1788571696.270571:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384100.
00010000:00000010:1.1:1788571696.270582:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a200.
00010000:00000010:1.1:1788571696.290573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49400.
00010000:00000010:1.1:1788571696.343595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b000.
00000080:00000010:1.1:1788571696.465586:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622a940.
00000080:00000010:1.1:1788571696.465608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ee80.
00010000:00000010:1.1:1788571696.465634:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384900.
00010000:00000010:1.1:1788571696.465641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a400.
00010000:00000010:1.1:1788571696.485674:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384200.
00010000:00000010:1.1:1788571696.485694:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434400.
00010000:00000010:1.1:1788571697.122468:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b600.
00010000:00000010:1.1:1788571697.122493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435c00.
00000080:00000010:1.1:1788571697.138470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342308840.
00000080:00000010:1.1:1788571697.138491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e780.
00010000:00000010:1.1:1788571697.164477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b000.
00000080:00000010:1.1:1788571697.526336:0:1046:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221eda0.
00000080:00000010:1.1:1788571697.526387:0:1046:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423098c0.
00010000:00000010:1.1:1788571697.548467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a800.
00010000:00000010:1.1:1788571698.186516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ae00.
00010000:00000010:1.1:1788571698.186536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:1.1:1788571698.196717:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953462339c0.
00000080:00000010:1.1:1788571698.196738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c720.
00010000:00000010:1.1:1788571698.425710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48400.
00010000:00000010:1.1:1788571698.863503:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:1.1:1788571698.863520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c800.
00000080:00000010:1.1:1788571698.989465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ef80.
00000080:00000010:1.1:1788571698.989509:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3a100.
00010000:00000010:1.1:1788571698.989517:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386600.
00010000:00000010:1.1:1788571698.989524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350e0c600.
00000080:00000010:1.1:1788571699.625857:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a38000.
00000080:00000010:1.1:1788571699.625877:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c020.
00010000:00000010:1.1:1788571701.927525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325900.
00010000:00000010:1.1:1788571701.927536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00010000:00000010:1.1:1788571702.005561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:1.1:1788571702.028858:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325000.
00010000:00000010:1.1:1788571702.028878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:1.1:1788571702.038623:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342110000.
00000080:00000010:1.1:1788571702.038645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c7a0.
00010000:00000010:1.1:1788571702.038658:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325300.
00010000:00000010:1.1:1788571702.038665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788571702.052292:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649b80.
00000080:00000010:1.1:1788571702.056777:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342111080.
00010000:00000010:1.1:1788571702.056787:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:1.1:1788571702.056793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98e00.
00000080:00000010:1.1:1788571702.069198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516495e0.
00000080:00000010:1.1:1788571702.069232:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546d9080.
00010000:00000010:1.1:1788571702.069240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ef00.
00010000:00000010:1.1:1788571702.069246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:1.1:1788571702.074708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649b80.
00000080:00000010:1.1:1788571702.074744:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546ddac0.
00010000:00000010:1.1:1788571702.592633:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ed00.
00010000:00000010:1.1:1788571702.592648:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f0b400.
00010000:00000010:1.1:1788571703.554556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34cc00.
00010000:00000010:1.1:1788571703.578016:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386e00.
00010000:00000010:1.1:1788571703.578029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ecc00.
00000080:00000010:1.1:1788571703.585559:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230a940.
00000080:00000010:1.1:1788571703.585574:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221eee0.
00010000:00000010:1.1:1788571703.594669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a600.
00000080:00000010:1.1:1788571703.675617:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230f380.
00000080:00000010:1.1:1788571703.675625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e6c0.
00010000:00000010:1.1:1788571703.679801:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386900.
00010000:00000010:1.1:1788571703.679810:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516ed600.
00000080:00000010:1.1:1788571703.690746:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649140.
00000080:00000010:1.1:1788571703.690761:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423098c0.
00010000:00000010:1.1:1788571703.690767:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386200.
00010000:00000010:1.1:1788571703.690772:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953516edc00.
00000080:00000010:1.1:1788571703.700681:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516498e0.
00000080:00000010:1.1:1788571703.700700:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230d280.
00010000:00000010:1.1:1788571703.708614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ac00.
00000080:00000010:1.1:1788571704.102754:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655450c0.
00000080:00000010:1.1:1788571704.102784:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421118c0.
00010000:00000010:1.1:1788571704.102790:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8500.
00010000:00000010:1.1:1788571704.102795:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00010000:00000010:1.1:1788571705.036600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031000.
00010000:00000010:1.1:1788571706.002439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e500.
00010000:00000010:1.1:1788571706.002450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198400.
00010000:00000010:1.1:1788571706.187624:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4de500.
00010000:00000010:1.1:1788571706.187634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198200.
00010000:00000010:1.1:1788571706.214446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34dc00.
00010000:00000010:1.1:1788571706.389605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c000.
00010000:00000010:1.1:1788571706.432577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4dee00.
00010000:00000010:1.1:1788571706.432589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199a00.
00010000:00000010:1.1:1788571706.944605:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e600.
00010000:00000010:1.1:1788571706.944619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cd200.
00010000:00000010:1.1:1788571706.997563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d000.
00000080:00000010:1.1:1788571707.833441:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e18c0.
00000080:00000010:1.1:1788571707.833450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e3c0.
00000080:00000010:1.1:1788571708.115797:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ebe0.
00000080:00000010:1.1:1788571708.115827:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3e300.
00010000:00000010:1.1:1788571708.122843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031000.
00010000:00000010:1.1:1788571708.150490:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325500.
00010000:00000010:1.1:1788571708.150504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434200.
00000080:00000010:1.1:1788571708.168907:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3c200.
00000080:00000010:1.1:1788571708.168917:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649e40.
00010000:00000010:1.1:1788571708.168925:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325700.
00010000:00000010:1.1:1788571708.168929:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435a00.
00000080:00000010:1.1:1788571708.181339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649e20.
00000080:00000010:1.1:1788571708.181367:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3d280.
00010000:00000010:1.1:1788571708.184453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1400.
00010000:00000010:1.1:1788571708.358814:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325a00.
00010000:00000010:1.1:1788571708.358832:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434e00.
00010000:00000010:3.1:1788571708.651539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7e00.
00010000:00000010:3.1:1788571708.651553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031e00.
00000080:00000010:3.1:1788571708.664498:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526adac0.
00000080:00000010:3.1:1788571708.664515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e4a0.
00010000:00000010:3.1:1788571708.669750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0ca800.
00000080:00000010:1.1:1788571708.803441:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526aa100.
00000080:00000010:1.1:1788571708.803455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e760.
00000080:00000010:1.1:1788571709.334317:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ec80.
00000080:00000010:1.1:1788571709.334356:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e0000.
00010000:00000010:1.1:1788571709.338501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030200.
00010000:00000010:3.1:1788571709.522477:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2b00.
00010000:00000010:3.1:1788571709.522489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0ca200.
00000080:00000010:3.1:1788571709.540987:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e3180.
00000080:00000010:3.1:1788571709.540999:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ec00.
00010000:00000010:3.1:1788571709.550578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0ca200.
00010000:00000010:0.1:1788571710.162151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ae00.
00000080:00000010:0.1:1788571710.166591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545a20.
00000080:00000010:0.1:1788571710.166615:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346235280.
00010000:00000010:0.1:1788571710.212143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1600.
00000080:00000010:3.1:1788571710.305104:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e480.
00000080:00000010:3.1:1788571710.305135:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953462339c0.
00010000:00000010:3.1:1788571710.305143:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a400.
00010000:00000010:3.1:1788571710.305148:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a400.
00000080:00000010:3.1:1788571710.305152:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c200.
00000080:00000010:3.1:1788571710.305166:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346236300.
00010000:00000010:3.1:1788571710.306954:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ae00.
00010000:00000010:3.1:1788571710.306969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b800.
00000080:00000010:3.1:1788571710.325746:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c440.
00000080:00000010:0.1:1788571710.335453:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346234200.
00010000:00000010:0.1:1788571710.417485:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a200.
00010000:00000010:0.1:1788571710.417501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ac00.
00000080:00000010:0.1:1788571710.429143:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342308840.
00000080:00000010:0.1:1788571710.429158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e3e0.
00010000:00000010:0.1:1788571710.441580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1800.
00000080:00000010:3.1:1788571710.585484:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342308000.
00000080:00000010:3.1:1788571710.585495:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e060.
00010000:00000010:3.1:1788571710.601057:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661600.
00000080:00000010:0.1:1788571710.694454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221eb80.
00000080:00000010:0.1:1788571710.698831:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622d280.
00010000:00000010:0.1:1788571710.698842:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0e00.
00010000:00000010:0.1:1788571710.698846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654df600.
00000080:00000010:0.1:1788571710.704964:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655458a0.
00000080:00000010:0.1:1788571710.704992:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622a100.
00010000:00000010:0.1:1788571710.705000:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0400.
00010000:00000010:0.1:1788571710.705005:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dfc00.
00000080:00000010:0.1:1788571710.711819:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655450e0.
00000080:00000010:0.1:1788571710.711844:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346228000.
00010000:00000010:0.1:1788571710.720657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0400.
00010000:00000010:0.1:1788571710.813428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8700.
00010000:00000010:0.1:1788571710.813435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dfa00.
00010000:00000010:3.1:1788571711.058609:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0600.
00010000:00000010:3.1:1788571711.058621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57600.
00000080:00000010:3.1:1788571711.102600:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346229080.
00000080:00000010:3.1:1788571711.102607:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e880.
00010000:00000010:3.1:1788571711.133432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c800.
00010000:00000010:3.1:1788571711.263441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d200.
00000080:00000010:3.1:1788571711.303642:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a39080.
00000080:00000010:3.1:1788571711.303655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221eee0.
00000080:00000010:3.1:1788571711.433456:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420f98c0.
00000080:00000010:3.1:1788571711.433468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e420.
00010000:00000010:3.1:1788571711.450057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661600.
00010000:00000010:3.1:1788571711.579558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ed00.
00010000:00000010:3.1:1788571711.579569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2bc00.
00000080:00000010:3.1:1788571711.588499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420fa100.
00000080:00000010:3.1:1788571711.588515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e6c0.
00010000:00000010:3.1:1788571711.593525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661600.
00010000:00000010:3.1:1788571711.605528:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e900.
00010000:00000010:3.1:1788571711.605542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2a200.
00000080:00000010:3.1:1788571711.613551:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420fb9c0.
00000080:00000010:3.1:1788571711.613565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c9e0.
00010000:00000010:3.1:1788571711.617698:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e100.
00010000:00000010:3.1:1788571711.617704:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2a000.
00000080:00000010:3.1:1788571711.626477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c360.
00000080:00000010:3.1:1788571711.626506:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420fdac0.
00010000:00000010:3.1:1788571711.631537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661600.
00000080:00000010:3.1:1788571711.759287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221efa0.
00000080:00000010:3.1:1788571711.759316:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420f9080.
00010000:00000010:3.1:1788571711.759322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e500.
00010000:00000010:3.1:1788571711.759327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2a800.
00010000:00000010:0.1:1788571711.847620:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155400.
00010000:00000010:0.1:1788571711.847628:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2ac00.
00000080:00000010:0.1:1788571712.112293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ef60.
00000080:00000010:0.1:1788571712.112326:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526adac0.
00010000:00000010:0.1:1788571712.132844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0600.
00000080:00000010:1.1:1788571712.342341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e2a0.
00000080:00000010:1.1:1788571712.342362:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526aca40.
00010000:00000010:1.1:1788571712.349713:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7d00.
00010000:00000010:1.1:1788571712.349721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cba00.
00010000:00000010:1.1:1788571712.372983:0:197836:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34da00.
00010000:00000010:1.1:1788571712.405139:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7900.
00010000:00000010:1.1:1788571712.405151:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cbe00.
00000080:00000010:3.1:1788571712.417548:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526a98c0.
00000080:00000010:3.1:1788571712.417563:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e100.
00010000:00000010:3.1:1788571712.420587:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ac00.
00010000:00000010:3.1:1788571712.616517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ac00.
00010000:00000010:3.1:1788571712.643665:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384200.
00010000:00000010:3.1:1788571712.643681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0caa00.
00000080:00000010:3.1:1788571712.665690:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526ac200.
00000080:00000010:3.1:1788571712.665703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ea80.
00010000:00000010:3.1:1788571712.691343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ac00.
00000080:00000010:1.1:1788571712.732424:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620f380.
00000080:00000010:1.1:1788571712.732433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8ce80.
00010000:00000010:1.1:1788571712.732443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384d00.
00010000:00000010:1.1:1788571712.732448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cc800.
00010000:00000010:3.1:1788571712.836536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384300.
00010000:00000010:3.1:1788571712.836550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cce00.
00000080:00000010:3.1:1788571712.869437:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221efe0.
00000080:00000010:3.1:1788571712.869797:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620eb40.
00010000:00000010:3.1:1788571712.869805:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384b00.
00010000:00000010:3.1:1788571712.869809:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cd000.
00000080:00000010:3.1:1788571712.875297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8ccc0.
00000080:00000010:3.1:1788571712.875323:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346208840.
00010000:00000010:3.1:1788571712.881690:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2200.
00010000:00000010:3.1:1788571712.881710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cda00.
00000080:00000010:3.1:1788571712.899639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c5c0.
00000080:00000010:3.1:1788571712.899659:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e18c0.
00010000:00000010:0.1:1788571712.981530:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2600.
00010000:00000010:0.1:1788571712.981536:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cc000.
00000080:00000010:1.1:1788571712.987651:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e6b40.
00000080:00000010:1.1:1788571712.987666:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e180.
00010000:00000010:3.1:1788571713.045032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d200.
00010000:00000010:3.1:1788571713.168552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2b00.
00010000:00000010:3.1:1788571713.168563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cc400.
00000080:00000010:3.1:1788571713.174718:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e1080.
00000080:00000010:3.1:1788571713.174730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e2e0.
00010000:00000010:3.1:1788571713.182448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a200.
00010000:00000010:3.1:1788571713.303551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4bc00.
00010000:00000010:3.1:1788571713.303564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522ccc00.
00000080:00000010:3.1:1788571713.322770:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e39c0.
00000080:00000010:3.1:1788571713.322781:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ee80.
00010000:00000010:3.1:1788571713.327558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4bb00.
00010000:00000010:3.1:1788571713.327574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198e00.
00000080:00000010:3.1:1788571713.342496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c480.
00000080:00000010:3.1:1788571713.342518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e2100.
00010000:00000010:3.1:1788571713.345663:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b600.
00010000:00000010:3.1:1788571713.345679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199000.
00000080:00000010:3.1:1788571713.352843:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c760.
00000080:00000010:3.1:1788571713.353091:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e5280.
00010000:00000010:3.1:1788571713.357582:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4be00.
00010000:00000010:3.1:1788571713.357597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199a00.
00000080:00000010:3.1:1788571713.438708:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e3180.
00010000:00000010:3.1:1788571713.445576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384c00.
00010000:00000010:3.1:1788571713.445592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199c00.
00000080:00000010:3.1:1788571713.450575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c760.
00000080:00000010:3.1:1788571713.450602:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620c200.
00010000:00000010:3.1:1788571713.450608:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b500.
00010000:00000010:3.1:1788571713.450614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199600.
00000080:00000010:3.1:1788571713.471932:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cd80.
00000080:00000010:3.1:1788571713.471950:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e0000.
00010000:00000010:3.1:1788571713.471956:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384f00.
00010000:00000010:3.1:1788571713.471961:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198400.
00000080:00000010:3.1:1788571713.475500:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c760.
00000080:00000010:3.1:1788571713.479696:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620e300.
00010000:00000010:3.1:1788571713.479712:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b200.
00010000:00000010:3.1:1788571713.479717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199800.
00010000:00000010:3.1:1788571713.576740:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384600.
00010000:00000010:3.1:1788571713.576747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199200.
00000080:00000010:3.1:1788571713.582419:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620dac0.
00000080:00000010:3.1:1788571713.582431:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ef20.
00010000:00000010:3.1:1788571713.588526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a200.
00000080:00000010:0.1:1788571713.653942:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e780.
00000080:00000010:0.1:1788571713.653968:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346231080.
00010000:00000010:0.1:1788571713.660553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0e00.
00000080:00000010:0.1:1788571713.786164:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474ca40.
00010000:00000010:0.1:1788571713.818419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1a00.
00010000:00000010:3.1:1788571714.003512:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:3.1:1788571714.003525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5200.
00000080:00000010:3.1:1788571714.011649:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474b180.
00000080:00000010:3.1:1788571714.011664:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e560.
00010000:00000010:3.1:1788571714.011681:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4d00.
00010000:00000010:3.1:1788571714.011686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4c00.
00000080:00000010:3.1:1788571714.022389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c780.
00000080:00000010:3.1:1788571714.026575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474c200.
00010000:00000010:3.1:1788571714.026588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b000.
00010000:00000010:3.1:1788571714.026594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5800.
00000080:00000010:3.1:1788571714.044435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c5a0.
00000080:00000010:3.1:1788571714.044560:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e1080.
00010000:00000010:3.1:1788571714.044567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3900.
00010000:00000010:3.1:1788571714.044571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4400.
00010000:00000010:1.1:1788571714.250476:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:1.1:1788571714.250489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661600.
00010000:00000010:1.1:1788571714.348677:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ce00.
00010000:00000010:0.1:1788571714.364694:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3100.
00010000:00000010:0.1:1788571714.364699:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661a00.
00000080:00000010:0.1:1788571714.379486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e7380.
00000080:00000010:0.1:1788571714.379499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ef80.
00010000:00000010:0.1:1788571714.383540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0e00.
00000080:00000010:3.1:1788571714.411697:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e18c0.
00000080:00000010:3.1:1788571714.411706:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545d20.
00010000:00000010:3.1:1788571714.411713:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3000.
00010000:00000010:3.1:1788571714.411718:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660e00.
00010000:00000010:1.1:1788571714.412556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3400.
00010000:00000010:1.1:1788571714.412564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661000.
00000080:00000010:3.1:1788571714.423789:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c5a0.
00000080:00000010:3.1:1788571714.423807:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e5280.
00010000:00000010:3.1:1788571714.432484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a200.
00010000:00000010:3.1:1788571714.439834:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7100.
00010000:00000010:3.1:1788571714.439840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660600.
00000080:00000010:3.1:1788571714.454663:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e2100.
00000080:00000010:3.1:1788571714.454671:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cdc0.
00010000:00000010:3.1:1788571714.454677:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7a00.
00010000:00000010:3.1:1788571714.454681:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660a00.
00000080:00000010:3.1:1788571714.463562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c140.
00000080:00000010:3.1:1788571714.463582:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e39c0.
00010000:00000010:3.1:1788571714.470960:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7400.
00010000:00000010:3.1:1788571714.470975:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661400.
00000080:00000010:3.1:1788571714.480520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c5a0.
00000080:00000010:3.1:1788571714.480539:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e6300.
00000080:00000010:0.1:1788571714.484547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c140.
00000080:00000010:0.1:1788571714.484572:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e0000.
00010000:00000010:0.1:1788571714.484578:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7f00.
00010000:00000010:0.1:1788571714.484583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661200.
00010000:00000010:3.1:1788571714.484671:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7200.
00010000:00000010:3.1:1788571714.484676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660800.
00000080:00000010:1.1:1788571714.497590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655455e0.
00000080:00000010:1.1:1788571714.497615:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230a940.
00010000:00000010:1.1:1788571714.497622:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7e00.
00010000:00000010:1.1:1788571714.497627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660c00.
00000080:00000010:1.1:1788571714.510243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516496e0.
00000080:00000010:1.1:1788571714.510277:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230ca40.
00010000:00000010:1.1:1788571714.513929:0:197833:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:1.1:1788571714.513941:0:197833:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661800.
00000080:00000010:1.1:1788571714.520902:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516498a0.
00000080:00000010:1.1:1788571714.524004:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474a100.
00010000:00000010:1.1:1788571714.531508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030800.
00010000:00000010:0.1:1788571714.549674:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788571714.549687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dee00.
00000080:00000010:1.1:1788571714.583853:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474d280.
00000080:00000010:1.1:1788571714.583861:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516496e0.
00010000:00000010:1.1:1788571714.598759:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030200.
00010000:00000010:3.1:1788571714.686480:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:3.1:1788571714.686494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dfa00.
00000080:00000010:3.1:1788571714.696429:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474f380.
00000080:00000010:3.1:1788571714.696606:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ef00.
00010000:00000010:3.1:1788571714.712816:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a200.
00000080:00000010:3.1:1788571714.744767:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534209dac0.
00000080:00000010:3.1:1788571714.744780:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c5a0.
00010000:00000010:3.1:1788571714.756608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a200.
00010000:00000010:3.1:1788571714.881177:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c600.
00010000:00000010:3.1:1788571715.119803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c600.
00010000:00000010:3.1:1788571715.131806:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ea00.
00010000:00000010:3.1:1788571715.131813:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165aa00.
00000080:00000010:3.1:1788571715.131819:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342308000.
00000080:00000010:3.1:1788571715.131825:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221eda0.
00010000:00000010:3.1:1788571715.131837:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e600.
00010000:00000010:3.1:1788571715.131840:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ba00.
00000080:00000010:3.1:1788571715.146335:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c4c0.
00010000:00000010:3.1:1788571715.230460:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ee00.
00010000:00000010:3.1:1788571715.230466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d400.
00010000:00000010:3.1:1788571715.283200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e300.
00010000:00000010:3.1:1788571715.283213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ca00.
00000080:00000010:3.1:1788571715.306311:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342308840.
00000080:00000010:3.1:1788571715.306326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ef20.
00010000:00000010:3.1:1788571715.314132:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e200.
00010000:00000010:3.1:1788571715.314141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d000.
00000080:00000010:3.1:1788571715.317632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8ca00.
00000080:00000010:3.1:1788571715.317659:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230eb40.
00010000:00000010:3.1:1788571715.317665:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8600.
00010000:00000010:3.1:1788571715.317670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c000.
00000080:00000010:3.1:1788571715.331455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c1a0.
00000080:00000010:3.1:1788571715.331481:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342098840.
00010000:00000010:3.1:1788571715.331487:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ef00.
00010000:00000010:3.1:1788571715.331492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34dc00.
00000080:00000010:3.1:1788571715.362450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e780.
00000080:00000010:3.1:1788571715.362489:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346234200.
00010000:00000010:3.1:1788571715.370227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0900.
00010000:00000010:3.1:1788571715.370244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d800.
00000080:00000010:3.1:1788571715.408837:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c1a0.
00000080:00000010:3.1:1788571715.408856:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346236300.
00010000:00000010:3.1:1788571715.476479:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0600.
00010000:00000010:3.1:1788571715.476488:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34de00.
00000080:00000010:3.1:1788571715.487187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ee80.
00000080:00000010:3.1:1788571715.487223:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342115ac0.
00010000:00000010:3.1:1788571715.487230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8e00.
00010000:00000010:3.1:1788571715.487235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a000.
00000080:00000010:3.1:1788571715.502720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c1a0.
00000080:00000010:3.1:1788571715.502744:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534209b180.
00010000:00000010:3.1:1788571715.516657:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ba00.
00000080:00000010:3.1:1788571715.528435:0:197833:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421139c0.
00000080:00000010:3.1:1788571715.528459:0:197833:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c7e0.
00010000:00000010:3.1:1788571715.533413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ba00.
00010000:00000010:3.1:1788571715.556825:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0b00.
00010000:00000010:3.1:1788571715.556840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a400.
00000080:00000010:0.1:1788571715.567439:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342112100.
00000080:00000010:0.1:1788571715.567464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c1a0.
00010000:00000010:0.1:1788571715.674165:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0700.
00010000:00000010:0.1:1788571715.674176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b400.
00000080:00000010:0.1:1788571715.723518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421118c0.
00000080:00000010:0.1:1788571715.723534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ec60.
00010000:00000010:0.1:1788571715.723551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0000.
00000080:00000010:3.1:1788571715.876569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e0e0.
00000080:00000010:3.1:1788571715.876581:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346232100.
00010000:00000010:3.1:1788571719.127863:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0300.
00010000:00000010:3.1:1788571719.127877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34cc00.
00000080:00000010:3.1:1788571719.149435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ee80.
00010000:00000010:0.1:1788571719.286042:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0000.
00010000:00000010:0.1:1788571719.286058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d200.
00010000:00000010:3.1:1788571719.499764:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0f00.
00010000:00000010:3.1:1788571719.499779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34da00.
00000080:00000010:3.1:1788571719.587456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e520.
00000080:00000010:1.1:1788571719.786593:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221eda0.
00010000:00000010:1.1:1788571719.820512:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb300.
00010000:00000010:1.1:1788571719.820521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030600.
00010000:00000010:3.1:1788571719.871230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8d00.
00010000:00000010:3.1:1788571719.871240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c400.
00000080:00000010:1.1:1788571719.884573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ef00.
00010000:00000010:1.1:1788571719.900496:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb300.
00010000:00000010:1.1:1788571719.900513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030000.
00010000:00000010:0.1:1788571720.089616:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155200.
00010000:00000010:0.1:1788571720.089628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0200.
00000080:00000010:0.1:1788571720.136689:0:197998:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e1c0.
00010000:00000010:1.1:1788571720.146268:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155400.
00010000:00000010:1.1:1788571720.146293:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0c00.
00000080:00000010:1.1:1788571720.155443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545b60.
00010000:00000010:0.1:1788571720.171868:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155500.
00010000:00000010:0.1:1788571720.171885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030600.
00000080:00000010:3.1:1788571720.198695:0:197998:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516498c0.
00010000:00000010:3.1:1788571720.198711:0:197998:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8d00.
00010000:00000010:3.1:1788571720.198715:0:197998:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b000.
00010000:00000010:3.1:1788571720.388657:0:197998:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e200.
00010000:00000010:3.1:1788571720.388669:0:197998:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ca00.
00000080:00000010:3.1:1788571720.416217:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ed80.
00000080:00000010:0.1:1788571720.458606:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cd60.
00010000:00000010:1.1:1788571720.484444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155500.
00010000:00000010:1.1:1788571720.484451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0c00.
00010000:00000010:1.1:1788571720.699644:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb000.
00010000:00000010:1.1:1788571720.699652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660800.
00000080:00000010:1.1:1788571720.849447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221efe0.
00010000:00000010:1.1:1788571720.865488:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb000.
00010000:00000010:1.1:1788571720.865504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031a00.
00000080:00000010:0.1:1788571721.226616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221eee0.
00010000:00000010:0.1:1788571721.247875:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155600.
00010000:00000010:0.1:1788571721.247892:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1000.
00000080:00000010:0.1:1788571721.261448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545c80.
00010000:00000010:1.1:1788571721.279451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155600.
00010000:00000010:1.1:1788571721.279468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0a00.
00010000:00000010:3.1:1788571721.430526:0:197998:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8900.
00010000:00000010:3.1:1788571721.430533:0:197998:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5e00.
00000080:00000010:3.1:1788571721.449830:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e100.
00010000:00000010:3.1:1788571721.463436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8900.
00010000:00000010:3.1:1788571721.463453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b400.
00000080:00000010:3.1:1788571721.644568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e500.
00010000:00000010:3.1:1788571721.657632:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8e00.
00010000:00000010:3.1:1788571721.657649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b600.
00000080:00000010:3.1:1788571721.691599:0:197998:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c800.
00010000:00000010:0.1:1788571721.704437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155400.
00010000:00000010:0.1:1788571721.704462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165be00.
00000080:00000010:3.1:1788571721.729977:0:197998:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cf60.
00010000:00000010:3.1:1788571721.738647:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8600.
00010000:00000010:3.1:1788571721.738655:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a800.
00010000:00000010:3.1:1788571721.946642:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8600.
00010000:00000010:3.1:1788571721.946649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4800.
00010000:00000010:1.1:1788571721.992971:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b100.
00010000:00000010:1.1:1788571721.992988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4600.
00010000:00000010:0.1:1788571722.451508:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155b00.
00010000:00000010:0.1:1788571722.451524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1000.
00000080:00000010:3.1:1788571722.497835:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545ce0.
00000080:00000010:1.1:1788571722.545197:0:197998:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e3c0.
00010000:00000010:1.1:1788571722.551595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb200.
00010000:00000010:1.1:1788571722.551602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030200.
00000080:00000010:1.1:1788571722.574437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649760.
00010000:00000010:1.1:1788571722.609514:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb200.
00010000:00000010:1.1:1788571722.609523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030200.
00000080:00000010:3.1:1788571722.692440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ec00.
00000080:00000010:1.1:1788571722.766098:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8ca20.
00010000:00000010:0.1:1788571722.796430:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155e00.
00010000:00000010:0.1:1788571722.796445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031200.
00000080:00000010:0.1:1788571722.859808:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649420.
00010000:00000010:3.1:1788571722.897525:0:197998:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8c00.
00010000:00000010:3.1:1788571722.897542:0:197998:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0600.
00010000:00000010:3.1:1788571722.955475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8c00.
00010000:00000010:3.1:1788571722.955491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cde00.
00000080:00000010:3.1:1788571723.003447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e540.
00010000:00000010:3.1:1788571723.025067:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8200.
00010000:00000010:3.1:1788571723.025083:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165be00.
00000080:00000010:3.1:1788571723.062712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8ce00.
00010000:00000010:3.1:1788571723.085175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8000.
00010000:00000010:3.1:1788571723.085190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b800.
00000080:00000010:3.1:1788571723.119530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cde0.
00010000:00000010:3.1:1788571723.183314:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8100.
00010000:00000010:3.1:1788571723.183326:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ba00.
00000080:00000010:3.1:1788571723.296002:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e540.
00000080:00000010:0.1:1788571723.436450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e3c0.
00010000:00000010:0.1:1788571723.586452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155e00.
00010000:00000010:0.1:1788571723.586467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0000.
00000080:00000010:0.1:1788571723.648603:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221ed80.
00010000:00000010:0.1:1788571723.654616:0:197998:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155900.
00010000:00000010:0.1:1788571723.654624:0:197998:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0400.
00010000:00000010:0.1:1788571723.939455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155900.
00010000:00000010:0.1:1788571723.939464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2aa00.
00000080:00000010:1.1:1788571724.004471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80ca0.
00010000:00000010:0.1:1788571724.436421:0:198027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cde00.
00010000:00000010:3.1:1788571724.463840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0400.
00010000:00000010:0.1:1788571724.528580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434600.
00010000:00000010:1.1:1788571724.593189:0:198033:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435000.
00010000:00000010:0.1:1788571724.638703:0:197848:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435000.
00010000:00000010:3.1:1788571724.688575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1e00.
00010000:00000010:0.1:1788571724.746346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56a00.
00010000:00000010:0.1:1788571724.882148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0800.
00010000:00000010:0.1:1788571724.937610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0800.
00010000:00000010:0.1:1788571725.042156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56e00.
00010000:00000010:3.1:1788571725.105433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67a00.
00010000:00000010:0.1:1788571725.162428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67200.
00010000:00000010:0.1:1788571725.251779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66a00.
00010000:00000010:0.1:1788571725.270524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0800.
00010000:00000010:0.1:1788571725.324799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56000.
00010000:00000010:0.1:1788571725.333857:0:198061:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56200.
00010000:00000010:0.1:1788571725.357440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57600.
00010000:00000010:0.1:1788571725.400441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57800.
00010000:00000010:0.1:1788571725.480558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57800.
00010000:00000010:0.1:1788571725.504533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66600.
00010000:00000010:0.1:1788571725.533637:0:198074:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a000.
00010000:00000010:0.1:1788571725.558857:0:198073:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56200.
00010000:00000010:0.1:1788571725.617457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dfa00.
00010000:00000010:0.1:1788571725.632437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b600.
00010000:00000010:1.1:1788571725.669525:0:19:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56a00.
00010000:00000010:3.1:1788571725.716429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dfe00.
00010000:00000010:1.1:1788571725.875057:0:198088:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668a00.
00010000:00000010:1.1:1788571725.943438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67a00.
00010000:00000010:3.1:1788571725.986358:0:198093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669000.
00010000:00000010:0.1:1788571726.030960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669e00.
00010000:00000010:1.1:1788571726.065421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668200.
00010000:00000010:1.1:1788571726.117441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669600.
00010000:00000010:0.1:1788571726.180882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67a00.
00010000:00000010:1.1:1788571726.283526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ba00.
00010000:00000010:1.1:1788571726.362221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669800.
00010000:00000010:0.1:1788571726.447263:0:198110:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ba00.
00010000:00000010:0.1:1788571726.471426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a000.
00010000:00000010:1.1:1788571726.571751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669200.
00010000:00000010:0.1:1788571726.594592:0:198118:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ba00.
00010000:00000010:0.1:1788571726.619418:0:2363:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669400.
00010000:00000010:0.1:1788571726.652548:0:197848:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0ca800.
00010000:00000010:1.1:1788571726.707533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f200.
00010000:00000010:0.1:1788571726.748409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669400.
00010000:00000010:0.1:1788571726.783522:0:198125:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383ea00.
00000080:00000010:1.1:1788571750.096754:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e420.
00000080:00000010:1.1:1788571750.374282:0:199381:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655458e0.
00000080:00000010:3.1:1788571751.278174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545740.
00010000:00000010:3.1:1788571751.348449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:3.1:1788571751.348463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:3.1:1788571751.348467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:3.1:1788571751.348471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d049a00.
00010000:00000010:3.1:1788571751.348474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788571751.420262:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89535221e280.
00000080:00000010:1.1:1788571761.622522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cec0.
00010000:00000010:0.1:1788571863.217634:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:0.1:1788571863.217648:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:3.1:1788571863.244451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8d41a0.
00000080:00000010:3.1:1788571863.304310:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655458a0.
00000080:00000010:3.1:1788571863.343423:0:202990:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545b80.
00000080:00000010:3.1:1788571863.366988:0:75901:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351d78140.
00010000:00000010:3.1:1788571865.262446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59d700.
00010000:00000010:3.1:1788571865.262467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:1.1:1788571865.350428:0:203060:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d218300.
00010000:00000010:3.1:1788571865.419899:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788571865.419912:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52400.
00000080:00000010:3.1:1788571865.468545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8ce00.
00000080:00000010:0.1:1788571865.679960:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80a20.
00000080:00000010:0.1:1788571865.679980:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649f20.
00000080:00000010:0.1:1788571865.679992:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545720.
00000080:00000010:0.1:1788571865.680004:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c760.
00000080:00000010:0.1:1788571865.680010:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80340.
00000080:00000010:0.1:1788571865.680014:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f805e0.
00000080:00000010:0.1:1788571865.680019:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f803a0.
00000080:00000010:0.1:1788571865.680023:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80080.
00000080:00000010:0.1:1788571865.680026:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655452e0.
00000080:00000010:0.1:1788571865.680031:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649c60.
00000080:00000010:0.1:1788571865.680035:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545cc0.
00000080:00000010:0.1:1788571865.680039:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80d60.
00000080:00000010:0.1:1788571865.680042:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649ee0.
00000080:00000010:0.1:1788571865.680047:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8d4920.
00000080:00000010:0.1:1788571865.680051:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89536d2187a0.
00000080:00000010:0.1:1788571865.680055:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895348be8c80.
00000080:00000010:0.1:1788571865.680059:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c080.
00000080:00000010:0.1:1788571865.680064:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80760.
00000080:00000010:0.1:1788571865.680068:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f807a0.
00000080:00000010:0.1:1788571865.680074:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545c00.
00000080:00000010:0.1:1788571865.683613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545f80.
00000080:00000010:0.1:1788571865.683637:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c780.
00000080:00000010:0.1:1788571865.683646:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80c80.
00000080:00000010:0.1:1788571865.683667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649a80.
00000080:00000010:0.1:1788571865.688827:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f805c0.
00000080:00000010:0.1:1788571865.688866:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80ca0.
00000080:00000010:0.1:1788571865.688874:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c600.
00000080:00000010:0.1:1788571865.688882:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649300.
00000080:00000010:0.1:1788571865.688891:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655453e0.
00000080:00000010:0.1:1788571865.688897:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649200.
00000080:00000010:0.1:1788571865.688901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80720.
00000080:00000010:0.1:1788571865.688905:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545640.
00000080:00000010:0.1:1788571865.688909:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649ac0.
00000080:00000010:0.1:1788571865.688913:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80260.
00000080:00000010:0.1:1788571865.688916:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80700.
00000080:00000010:0.1:1788571865.688919:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f806c0.
00000080:00000010:0.1:1788571865.688922:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545980.
00000080:00000010:0.1:1788571865.688926:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649c20.
00000080:00000010:0.1:1788571865.688930:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80bc0.
00000080:00000010:0.1:1788571865.688939:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545a40.
00000080:00000010:0.1:1788571865.688942:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c000.
00000080:00000010:0.1:1788571865.688946:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cf20.
00000080:00000010:0.1:1788571865.688951:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cba0.
00000080:00000010:0.1:1788571865.688954:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655455e0.
00000080:00000010:0.1:1788571865.688958:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545d20.
00000080:00000010:0.1:1788571865.688962:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c040.
00000080:00000010:0.1:1788571865.688966:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cc80.
00000080:00000010:0.1:1788571865.688969:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655454a0.
00000080:00000010:0.1:1788571865.688975:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80a60.
00000080:00000010:0.1:1788571865.688979:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80980.
00000080:00000010:0.1:1788571865.688983:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545a20.
00000080:00000010:0.1:1788571865.688990:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80660.
00000080:00000010:0.1:1788571865.688994:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649120.
00000080:00000010:0.1:1788571865.689000:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f809a0.
00000080:00000010:0.1:1788571865.689003:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655454e0.
00000080:00000010:0.1:1788571865.689007:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649400.
00000080:00000010:0.1:1788571865.689012:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649480.
00000080:00000010:0.1:1788571865.689016:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80240.
00000080:00000010:0.1:1788571865.689020:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545b00.
00000080:00000010:0.1:1788571865.689046:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545020.
00000080:00000010:0.1:1788571865.689050:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545880.
00000080:00000010:0.1:1788571865.689054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545320.
00000080:00000010:0.1:1788571865.689058:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649d60.
00000080:00000010:0.1:1788571865.689062:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80ba0.
00000080:00000010:0.1:1788571865.689069:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545360.
00000080:00000010:0.1:1788571865.689073:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649ce0.
00000080:00000010:0.1:1788571865.689076:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cbe0.
00000080:00000010:0.1:1788571865.689081:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f804e0.
00000080:00000010:0.1:1788571865.689084:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c8a0.
00000080:00000010:0.1:1788571865.689087:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545180.
00000080:00000010:0.1:1788571865.689091:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80880.
00000080:00000010:0.1:1788571865.689095:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8cfc0.
00000080:00000010:0.1:1788571865.689098:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80fc0.
00000080:00000010:0.1:1788571865.689102:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80220.
00000080:00000010:0.1:1788571865.689105:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c740.
00000080:00000010:0.1:1788571865.689110:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c460.
00000080:00000010:0.1:1788571865.689114:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c100.
00000080:00000010:0.1:1788571865.689118:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649840.
00000080:00000010:0.1:1788571865.689122:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f806a0.
00000080:00000010:0.1:1788571865.689125:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80560.
00000080:00000010:0.1:1788571865.689129:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80800.
00000080:00000010:0.1:1788571865.689132:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80000.
00000080:00000010:0.1:1788571865.689623:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f807c0.
00000080:00000010:0.1:1788571865.689632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545660.
00000080:00000010:0.1:1788571865.689637:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80f60.
00000080:00000010:0.1:1788571865.689640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545ba0.
00000080:00000010:0.1:1788571865.689643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80840.
00000080:00000010:0.1:1788571865.689646:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895344f8c2e0.
00000080:00000010:0.1:1788571865.689650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649fc0.
00000080:00000010:0.1:1788571865.689653:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80ee0.
00000080:00000010:0.1:1788571865.689657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80060.
00000080:00000010:0.1:1788571865.689660:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80480.
00000080:00000010:0.1:1788571865.689664:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545840.
00000080:00000010:0.1:1788571865.689667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649ae0.
00000080:00000010:0.1:1788571865.689671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516494a0.
00000080:00000010:0.1:1788571865.689689:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80fe0.
00000080:00000010:0.1:1788571865.689693:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655456e0.
00000080:00000010:0.1:1788571865.689697:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895365545ca0.
00000080:00000010:0.1:1788571865.689700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649da0.
00000080:00000010:0.1:1788571865.689704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516497c0.
00000080:00000010:0.1:1788571865.689708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516490e0.
00000080:00000010:0.1:1788571865.689711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80d80.
00000080:00000010:0.1:1788571865.689715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953516499c0.
00000080:00000010:0.1:1788571865.689717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f803c0.
00000080:00000010:0.1:1788571865.689720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80180.
00000080:00000010:0.1:1788571865.689724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953655452c0.
00010000:00000010:0.1:1788571865.803808:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edbe00.
00010000:00000010:0.1:1788571865.803822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947d000.
00010000:00000010:0.1:1788571865.803827:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d049c00.
00010000:00000010:0.1:1788571865.803831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:0.1:1788571865.803836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542bc00.
00010000:00000010:0.1:1788571865.803848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654df200.
00010000:00000010:0.1:1788571865.803853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199000.
00010000:00000010:0.1:1788571865.803857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198000.
00010000:00000010:0.1:1788571865.803861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162800.
00010000:00000010:0.1:1788571865.803864:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162200.
00010000:00000010:0.1:1788571865.803882:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435000.
00010000:00000010:0.1:1788571865.803887:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434800.
00010000:00000010:0.1:1788571865.803890:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435200.
00010000:00000010:0.1:1788571865.803892:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435800.
00010000:00000010:0.1:1788571865.803895:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434c00.
00010000:00000010:0.1:1788571865.803898:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434400.
00010000:00000010:0.1:1788571865.803901:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59da00.
00010000:00000010:0.1:1788571865.803904:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:0.1:1788571865.803907:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59d300.
00010000:00000010:0.1:1788571865.803909:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:0.1:1788571865.803914:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:0.1:1788571865.803917:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52400.
00010000:00000010:0.1:1788571865.803920:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59d800.
00010000:00000010:0.1:1788571865.803939:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00010000:00000010:0.1:1788571865.803942:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59dc00.
00010000:00000010:0.1:1788571865.803945:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00010000:00000010:0.1:1788571865.803948:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59df00.
00010000:00000010:0.1:1788571865.803950:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:0.1:1788571865.803953:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:0.1:1788571865.803956:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947ca00.
00010000:00000010:0.1:1788571865.803959:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:0.1:1788571865.803962:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:0.1:1788571865.803965:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:0.1:1788571865.803967:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:0.1:1788571865.803969:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00010000:00000010:0.1:1788571865.803971:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00010000:00000010:0.1:1788571865.803974:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00010000:00000010:0.1:1788571865.803976:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:0.1:1788571865.803979:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947d600.
00010000:00000010:0.1:1788571865.803982:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00010000:00000010:0.1:1788571865.803986:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163e200.
00010000:00000010:0.1:1788571865.803989:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f800.
00010000:00000010:0.1:1788571865.803992:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947d200.
00010000:00000010:0.1:1788571865.803996:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48000.
00010000:00000010:0.1:1788571865.803998:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947c400.
00010000:00000010:0.1:1788571865.804001:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163f600.
00010000:00000010:0.1:1788571865.804004:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48c00.
00010000:00000010:0.1:1788571865.804007:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67e00.
00010000:00000010:0.1:1788571865.829841:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163ec00.
00010000:00000010:0.1:1788571865.829854:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163e800.
00010000:00000010:0.1:1788571865.829858:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163f400.
00010000:00000010:0.1:1788571865.829862:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49400.
00010000:00000010:0.1:1788571865.829865:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66a00.
00010000:00000010:0.1:1788571865.829869:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163ee00.
00010000:00000010:0.1:1788571865.829873:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48800.
00010000:00000010:0.1:1788571865.829875:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163fa00.
00010000:00000010:0.1:1788571865.834990:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f600.
00010000:00000010:0.1:1788571865.835006:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383fc00.
00010000:00000010:0.1:1788571865.835010:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48200.
00010000:00000010:0.1:1788571865.835013:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48a00.
00010000:00000010:0.1:1788571865.835015:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e000.
00010000:00000010:0.1:1788571865.835017:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383fa00.
00010000:00000010:0.1:1788571865.835020:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49a00.
00010000:00000010:0.1:1788571865.835023:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341518e00.
00010000:00000010:0.1:1788571865.835028:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341518c00.
00010000:00000010:0.1:1788571865.835031:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49600.
00010000:00000010:0.1:1788571865.835036:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341519000.
00010000:00000010:0.1:1788571865.835041:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67a00.
00010000:00000010:0.1:1788571865.835045:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669400.
00010000:00000010:0.1:1788571865.835047:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49c00.
00010000:00000010:0.1:1788571865.835050:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67c00.
00010000:00000010:0.1:1788571865.835053:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66e00.
00010000:00000010:0.1:1788571865.835056:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669600.
00010000:00000010:0.1:1788571865.835059:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cb000.
00010000:00000010:0.1:1788571865.835063:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0ca000.
00010000:00000010:0.1:1788571865.835066:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cae00.
00010000:00000010:0.1:1788571865.835069:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cb600.
00010000:00000010:0.1:1788571865.835072:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66c00.
00010000:00000010:0.1:1788571865.835074:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669200.
00010000:00000010:0.1:1788571865.835076:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0ca600.
00010000:00000010:0.1:1788571865.835080:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67600.
00010000:00000010:0.1:1788571865.835082:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cbe00.
00010000:00000010:0.1:1788571865.835084:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cbc00.
00010000:00000010:0.1:1788571865.835087:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0ca400.
00010000:00000010:0.1:1788571865.835089:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cb800.
00010000:00000010:0.1:1788571865.835092:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cac00.
00010000:00000010:0.1:1788571865.835096:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cb400.
00010000:00000010:0.1:1788571865.835098:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0caa00.
00010000:00000010:0.1:1788571865.835111:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66600.
00010000:00000010:0.1:1788571865.835130:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542aa00.
00010000:00000010:0.1:1788571865.835134:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654de200.
00010000:00000010:0.1:1788571865.835136:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dee00.
00010000:00000010:0.1:1788571865.835139:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dfe00.
00010000:00000010:0.1:1788571865.835143:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542be00.
00010000:00000010:0.1:1788571865.835145:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ae00.
00010000:00000010:0.1:1788571865.835148:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a200.
00010000:00000010:0.1:1788571865.835151:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b000.
00010000:00000010:0.1:1788571865.835155:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57600.
00010000:00000010:0.1:1788571865.835157:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654df800.
00010000:00000010:0.1:1788571865.835160:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dec00.
00010000:00000010:0.1:1788571865.835162:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a400.
00010000:00000010:0.1:1788571865.835165:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b800.
00010000:00000010:0.1:1788571865.835167:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a800.
00010000:00000010:0.1:1788571865.835171:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198e00.
00010000:00000010:0.1:1788571865.835174:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56a00.
00010000:00000010:0.1:1788571865.835176:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57800.
00010000:00000010:0.1:1788571865.835182:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199a00.
00010000:00000010:0.1:1788571865.835184:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199e00.
00010000:00000010:0.1:1788571865.835188:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163600.
00010000:00000010:0.1:1788571865.835192:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163200.
00010000:00000010:0.1:1788571865.835195:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162a00.
00010000:00000010:0.1:1788571865.835198:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163c00.
00010000:00000010:0.1:1788571865.835201:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654de800.
00010000:00000010:0.1:1788571865.835204:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654df400.
00010000:00000010:0.1:1788571865.835206:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dea00.
00010000:00000010:0.1:1788571865.835209:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654de600.
00010000:00000010:0.1:1788571865.835213:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cde00.
00010000:00000010:0.1:1788571865.835218:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1c00.
00010000:00000010:0.1:1788571865.835223:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162600.
00010000:00000010:0.1:1788571865.835226:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0800.
00010000:00000010:0.1:1788571865.835229:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1400.
00010000:00000010:0.1:1788571865.835232:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1e00.
00010000:00000010:0.1:1788571865.835240:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0c00.
00010000:00000010:0.1:1788571865.835244:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163400.
00010000:00000010:0.1:1788571865.835246:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382163800.
00010000:00000010:0.1:1788571865.835249:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435e00.
00010000:00000010:0.1:1788571865.835253:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198200.
00010000:00000010:0.1:1788571865.835256:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199c00.
00010000:00000010:0.1:1788571865.835259:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199600.
00010000:00000010:0.1:1788571865.835261:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198400.
00010000:00000010:0.1:1788571865.835264:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199800.
00010000:00000010:0.1:1788571865.835267:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434e00.
00010000:00000010:0.1:1788571865.835270:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198c00.
00010000:00000010:0.1:1788571865.835272:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199200.
00010000:00000010:0.1:1788571865.835275:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198800.
00010000:00000010:0.1:1788571865.835277:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d199400.
00010000:00000010:0.1:1788571865.835280:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198a00.
00010000:00000010:0.1:1788571865.835283:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d198600.
00010000:00000010:0.1:1788571865.835295:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435c00.
00010000:00000010:0.1:1788571865.835298:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435600.
00010000:00000010:0.1:1788571865.835301:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434600.
00010000:00000010:0.1:1788571865.835303:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0a00.
00010000:00000010:0.1:1788571865.835306:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435a00.
00010000:00000010:0.1:1788571865.835308:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365435400.
00010000:00000010:0.1:1788571865.835311:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434a00.
00010000:00000010:0.1:1788571865.835314:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030200.
00010000:00000010:0.1:1788571865.835319:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434000.
00010000:00000010:0.1:1788571865.835323:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2ba00.
00010000:00000010:0.1:1788571865.835327:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031200.
00010000:00000010:0.1:1788571865.835330:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2b600.
00010000:00000010:0.1:1788571865.835333:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2a200.
00010000:00000010:0.1:1788571865.835338:0:203065:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a500.
00010000:00000010:0.1:1788571865.835342:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2be00.
00010000:00000010:0.1:1788571865.835346:0:203065:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb200.
00010000:00000010:0.1:1788571865.835349:0:203065:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030600.
00010000:00000010:0.1:1788571865.835356:0:203065:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545af00.
00010000:00000010:0.1:1788571865.835384:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2b000.
00010000:00000010:0.1:1788571865.836678:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8900.
00010000:00000010:0.1:1788571865.836690:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2ae00.
00010000:00000010:0.1:1788571865.836696:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4dee00.
00010000:00000010:0.1:1788571865.836700:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2a600.
00010000:00000010:0.1:1788571865.836703:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4ded00.
00010000:00000010:0.1:1788571865.836707:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2ac00.
00010000:00000010:0.1:1788571865.836712:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4de500.
00010000:00000010:0.1:1788571865.836716:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2b400.
00010000:00000010:0.1:1788571865.836722:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e200.
00010000:00000010:0.1:1788571865.836730:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2a800.
00010000:00000010:0.1:1788571865.836734:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8600.
00010000:00000010:0.1:1788571865.836739:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2b200.
00010000:00000010:0.1:1788571865.836743:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8800.
00010000:00000010:0.1:1788571865.836747:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2b800.
00010000:00000010:0.1:1788571865.836752:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155000.
00010000:00000010:0.1:1788571865.836758:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1200.
00010000:00000010:0.1:1788571865.836762:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8d00.
00010000:00000010:0.1:1788571865.836767:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2a400.
00010000:00000010:0.1:1788571865.836770:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155e00.
00010000:00000010:0.1:1788571865.836774:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0000.
00010000:00000010:0.1:1788571865.836776:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8700.
00010000:00000010:0.1:1788571865.836781:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cc800.
00010000:00000010:0.1:1788571865.836784:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8100.
00010000:00000010:0.1:1788571865.836798:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cd200.
00010000:00000010:0.1:1788571865.836802:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8f00.
00010000:00000010:0.1:1788571865.836805:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cd400.
00010000:00000010:0.1:1788571865.836809:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8e00.
00010000:00000010:0.1:1788571865.836813:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cca00.
00010000:00000010:0.1:1788571865.836817:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8300.
00010000:00000010:0.1:1788571865.836821:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cc600.
00010000:00000010:0.1:1788571865.836831:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8000.
00010000:00000010:0.1:1788571865.836835:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cd000.
00010000:00000010:0.1:1788571865.836838:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953509a8500.
00010000:00000010:0.1:1788571865.836851:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cce00.
00010000:00000010:0.1:1788571865.836857:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b100.
00010000:00000010:0.1:1788571865.836861:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cda00.
00010000:00000010:0.1:1788571865.836866:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8000.
00010000:00000010:0.1:1788571865.836870:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b800.
00010000:00000010:0.1:1788571865.836874:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8200.
00010000:00000010:0.1:1788571865.836879:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165be00.
00010000:00000010:0.1:1788571865.836883:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8c00.
00010000:00000010:0.1:1788571865.836887:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ae00.
00010000:00000010:0.1:1788571865.836891:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8500.
00010000:00000010:0.1:1788571865.836894:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ac00.
00010000:00000010:0.1:1788571865.836897:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4ba00.
00010000:00000010:0.1:1788571865.836901:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cc000.
00010000:00000010:0.1:1788571865.836904:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4bb00.
00010000:00000010:0.1:1788571865.836907:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cdc00.
00010000:00000010:0.1:1788571865.836911:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4bc00.
00010000:00000010:0.1:1788571865.836916:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cc200.
00010000:00000010:0.1:1788571865.836922:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb600.
00010000:00000010:0.1:1788571865.836927:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030e00.
00010000:00000010:0.1:1788571865.836931:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b700.
00010000:00000010:0.1:1788571865.836935:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522ccc00.
00010000:00000010:0.1:1788571865.836939:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b600.
00010000:00000010:0.1:1788571865.836943:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cd600.
00010000:00000010:0.1:1788571865.836947:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155b00.
00010000:00000010:0.1:1788571865.836952:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1000.
00010000:00000010:0.1:1788571865.836955:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8600.
00010000:00000010:0.1:1788571865.836959:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a600.
00010000:00000010:0.1:1788571865.836962:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4be00.
00010000:00000010:0.1:1788571865.836966:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cc400.
00010000:00000010:0.1:1788571865.836975:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b400.
00010000:00000010:0.1:1788571865.836978:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953522cd800.
00010000:00000010:0.1:1788571865.836990:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b200.
00010000:00000010:0.1:1788571865.836997:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4800.
00010000:00000010:0.1:1788571865.837003:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b500.
00010000:00000010:0.1:1788571865.837008:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5e00.
00010000:00000010:0.1:1788571865.837012:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155a00.
00010000:00000010:0.1:1788571865.837019:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0e00.
00010000:00000010:0.1:1788571865.837022:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b300.
00010000:00000010:0.1:1788571865.837025:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4600.
00010000:00000010:0.1:1788571865.837027:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b900.
00010000:00000010:0.1:1788571865.837030:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5a00.
00010000:00000010:0.1:1788571865.837033:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4bf00.
00010000:00000010:0.1:1788571865.837037:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4e00.
00010000:00000010:0.1:1788571865.837042:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b800.
00010000:00000010:0.1:1788571865.837047:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5000.
00010000:00000010:0.1:1788571865.837049:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155400.
00010000:00000010:0.1:1788571865.837052:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1600.
00010000:00000010:0.1:1788571865.837055:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8b00.
00010000:00000010:0.1:1788571865.837058:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a800.
00010000:00000010:0.1:1788571865.837061:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4bd00.
00010000:00000010:0.1:1788571865.837063:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4a00.
00010000:00000010:0.1:1788571865.837065:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c4b000.
00010000:00000010:0.1:1788571865.837068:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5400.
00010000:00000010:0.1:1788571865.837073:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788571865.837076:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4c00.
00010000:00000010:0.1:1788571865.837080:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788571865.837084:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5200.
00010000:00000010:0.1:1788571865.837086:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788571865.837088:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5800.
00010000:00000010:0.1:1788571865.837091:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788571865.837093:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5600.
00010000:00000010:0.1:1788571865.837099:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8e00.
00010000:00000010:0.1:1788571865.837102:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b600.
00010000:00000010:0.1:1788571865.837104:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8900.
00010000:00000010:0.1:1788571865.837107:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b400.
00010000:00000010:0.1:1788571865.837110:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4d00.
00010000:00000010:0.1:1788571865.837113:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4400.
00010000:00000010:0.1:1788571865.837116:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4500.
00010000:00000010:0.1:1788571865.837118:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4200.
00010000:00000010:0.1:1788571865.837121:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788571865.837123:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a5c00.
00010000:00000010:0.1:1788571865.837125:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788571865.837128:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953797a4000.
00010000:00000010:0.1:1788571865.837130:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8700.
00010000:00000010:0.1:1788571865.837133:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b200.
00010000:00000010:0.1:1788571865.837135:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788571865.837140:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c400.
00010000:00000010:0.1:1788571865.837143:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788571865.837147:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660800.
00010000:00000010:0.1:1788571865.837149:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788571865.837152:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660400.
00010000:00000010:0.1:1788571865.837154:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788571865.837158:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661600.
00010000:00000010:0.1:1788571865.837160:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155500.
00010000:00000010:0.1:1788571865.837163:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1a00.
00010000:00000010:0.1:1788571865.837165:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788571865.837168:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660000.
00010000:00000010:0.1:1788571865.837171:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788571865.837173:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661c00.
00010000:00000010:0.1:1788571865.837178:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0900.
00010000:00000010:0.1:1788571865.837181:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660200.
00010000:00000010:0.1:1788571865.837184:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e800.
00010000:00000010:0.1:1788571865.837187:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661e00.
00010000:00000010:0.1:1788571865.837194:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e500.
00010000:00000010:0.1:1788571865.837196:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661a00.
00010000:00000010:0.1:1788571865.837199:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e000.
00010000:00000010:0.1:1788571865.837202:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660e00.
00010000:00000010:0.1:1788571865.837205:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb000.
00010000:00000010:0.1:1788571865.837208:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031a00.
00010000:00000010:0.1:1788571865.837211:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e100.
00010000:00000010:0.1:1788571865.837213:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661000.
00010000:00000010:0.1:1788571865.837216:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ea00.
00010000:00000010:0.1:1788571865.837219:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660a00.
00010000:00000010:0.1:1788571865.837221:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e900.
00010000:00000010:0.1:1788571865.837223:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660600.
00010000:00000010:0.1:1788571865.837226:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb500.
00010000:00000010:0.1:1788571865.837228:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031e00.
00010000:00000010:0.1:1788571865.837231:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ed00.
00010000:00000010:0.1:1788571865.837233:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661400.
00010000:00000010:0.1:1788571865.837235:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e400.
00010000:00000010:0.1:1788571865.837238:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661200.
00010000:00000010:0.1:1788571865.837240:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ec00.
00010000:00000010:0.1:1788571865.837242:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365660c00.
00010000:00000010:0.1:1788571865.837245:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e600.
00010000:00000010:0.1:1788571865.837247:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365661800.
00010000:00000010:0.1:1788571865.837250:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ee00.
00010000:00000010:0.1:1788571865.837253:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ca00.
00010000:00000010:0.1:1788571865.837255:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398e300.
00010000:00000010:0.1:1788571865.837257:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c600.
00010000:00000010:0.1:1788571865.837260:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8d00.
00010000:00000010:0.1:1788571865.837262:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165b000.
00010000:00000010:0.1:1788571865.837265:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534398ef00.
00010000:00000010:0.1:1788571865.837268:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d400.
00010000:00000010:0.1:1788571865.837271:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0b00.
00010000:00000010:0.1:1788571865.837274:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c000.
00010000:00000010:0.1:1788571865.837280:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0600.
00010000:00000010:0.1:1788571865.837284:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d000.
00010000:00000010:0.1:1788571865.837286:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8800.
00010000:00000010:0.1:1788571865.837289:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165bc00.
00010000:00000010:0.1:1788571865.837292:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0700.
00010000:00000010:0.1:1788571865.837294:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34dc00.
00010000:00000010:0.1:1788571865.837297:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0400.
00010000:00000010:0.1:1788571865.837299:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34de00.
00010000:00000010:0.1:1788571865.837302:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155200.
00010000:00000010:0.1:1788571865.837305:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0200.
00010000:00000010:0.1:1788571865.837307:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0e00.
00010000:00000010:0.1:1788571865.837310:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d800.
00010000:00000010:0.1:1788571865.837313:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0100.
00010000:00000010:0.1:1788571865.837316:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d600.
00010000:00000010:0.1:1788571865.837318:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0300.
00010000:00000010:0.1:1788571865.837321:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c200.
00010000:00000010:0.1:1788571865.837323:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb300.
00010000:00000010:0.1:1788571865.837325:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030000.
00010000:00000010:0.1:1788571865.837328:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8f00.
00010000:00000010:0.1:1788571865.837330:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a400.
00010000:00000010:0.1:1788571865.837333:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbb800.
00010000:00000010:0.1:1788571865.837336:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030800.
00010000:00000010:0.1:1788571865.837338:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0000.
00010000:00000010:0.1:1788571865.837341:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34c800.
00010000:00000010:0.1:1788571865.837343:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0f00.
00010000:00000010:0.1:1788571865.837346:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34cc00.
00010000:00000010:0.1:1788571865.837348:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0200.
00010000:00000010:0.1:1788571865.837350:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34da00.
00010000:00000010:0.1:1788571865.837353:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0c00.
00010000:00000010:0.1:1788571865.837356:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d200.
00010000:00000010:0.1:1788571865.837358:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8400.
00010000:00000010:0.1:1788571865.837361:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165aa00.
00010000:00000010:0.1:1788571865.837367:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbbd00.
00010000:00000010:0.1:1788571865.837369:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340030a00.
00010000:00000010:0.1:1788571865.837372:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0800.
00010000:00000010:0.1:1788571865.837374:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34ce00.
00010000:00000010:0.1:1788571865.837376:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0d00.
00010000:00000010:0.1:1788571865.837379:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895340031000.
00010000:00000010:3.1:1788571868.171495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00010000:00000010:0.1:1788571868.174511:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:3.1:1788571868.207289:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80fc0.
00000080:00000010:3.1:1788571868.214196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649480.
00010000:00000010:3.1:1788571868.214216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:1.1:1788571868.312878:0:203095:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00010000:00000010:3.1:1788571868.338851:0:203098:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:0.1:1788571868.348996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00000080:00000010:0.1:1788571868.349003:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895353a87ca0.
00010000:00000010:0.1:1788571868.363424:0:1377:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788571868.363438:0:1377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895379594860.
00010000:00000010:1.1:1788571868.402597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59df00.
00010000:00000010:1.1:1788571868.402611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:1.1:1788571868.402615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00010000:00000010:1.1:1788571868.402618:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59d600.
00010000:00000010:1.1:1788571868.402621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00010000:00000010:0.1:1788571868.403148:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab400.
00010000:00000010:0.1:1788571868.403155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:3.1:1788571869.345820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:3.1:1788571874.507598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00000080:00000010:1.1:1788571878.444690:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895348bd2e80.
00000080:00000010:3.1:1788571878.578949:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895343a6ec00.
00000080:00000010:3.1:1788571878.578965:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895353a875c0.
00000080:00000010:3.1:1788571878.649706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953795944c0.
00000080:00000010:3.1:1788571878.649719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895348bd2080.
00000080:00000010:1.1:1788571878.653783:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953797487a0.
00000080:00000010:1.1:1788571878.653804:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953524d0840.
00010000:00000010:0.1:1788571882.633548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59dc00.
00010000:00000010:0.1:1788571882.633566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788571896.270789:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbde0.
00010000:00000010:0.1:1788571896.270801:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59dd00.
00010000:00000010:0.1:1788571896.270804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:1.1:1788571896.373349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:1.1:1788571896.554614:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5200.
00000080:00000010:1.1:1788571896.554624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbde0.
00000080:00000010:1.1:1788571896.554630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895353a87440.
00000080:00000010:1.1:1788571896.554634:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895353a87cc0.
00000080:00000010:1.1:1788571896.554638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895351649f20.
00000080:00000010:1.1:1788571896.554643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895348bd2b20.
00000080:00000010:1.1:1788571896.554647:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f807a0.
00000080:00000010:1.1:1788571896.554654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350f80260.
00000080:00000010:1.1:1788571896.554658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895348be8ea0.
00010000:00000010:3.1:1788571896.564470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59d000.
00010000:00000010:3.1:1788571896.564485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00010000:00000010:3.1:1788571896.564489:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59dd00.
00010000:00000010:3.1:1788571896.564491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:3.1:1788571896.564494:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59d700.
00010000:00000010:3.1:1788571896.564496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00010000:00000010:3.1:1788571896.564499:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab100.
00010000:00000010:3.1:1788571896.564502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:3.1:1788571896.564504:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb400.
00010000:00000010:3.1:1788571896.564507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:3.1:1788571896.564509:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59de00.
00010000:00000010:3.1:1788571896.564512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00010000:00000010:3.1:1788571896.564514:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab500.
00010000:00000010:3.1:1788571896.564517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:3.1:1788571896.564831:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59db00.
00010000:00000010:3.1:1788571896.564837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00010000:00000010:3.1:1788571896.564840:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d2e5900.
00010000:00000010:3.1:1788571896.564855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00010000:00000010:3.1:1788571896.564870:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345edb800.
00010000:00000010:3.1:1788571896.564873:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00010000:00000010:3.1:1788571896.595015:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:3.1:1788571896.595027:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00010000:00000010:3.1:1788571896.595031:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325b00.
00010000:00000010:3.1:1788571896.595035:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947c600.
00010000:00000010:3.1:1788571896.595039:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386300.
00010000:00000010:3.1:1788571896.595042:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98e00.
00010000:00000010:3.1:1788571896.595045:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386500.
00010000:00000010:3.1:1788571896.595048:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49000.
00010000:00000010:3.1:1788571896.595051:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325a00.
00010000:00000010:3.1:1788571896.595054:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947ce00.
00010000:00000010:3.1:1788571896.595059:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953602ef900.
00010000:00000010:3.1:1788571896.595062:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f200.
00010000:00000010:3.1:1788571896.602432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325900.
00010000:00000010:3.1:1788571896.602442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947da00.
00010000:00000010:3.1:1788571896.602447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7900.
00010000:00000010:3.1:1788571896.602450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947dc00.
00010000:00000010:3.1:1788571896.602453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d0bb000.
00010000:00000010:3.1:1788571896.602457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163e400.
00010000:00000010:3.1:1788571896.602462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d3ce600.
00010000:00000010:3.1:1788571896.602560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163f800.
00010000:00000010:3.1:1788571896.602563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d386f00.
00010000:00000010:3.1:1788571896.602567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49800.
00010000:00000010:3.1:1788571896.602570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d3ce500.
00010000:00000010:3.1:1788571896.602573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163ea00.
00010000:00000010:3.1:1788571896.602576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d3ce300.
00010000:00000010:3.1:1788571896.602579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163f000.
00010000:00000010:3.1:1788571896.602584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384a00.
00010000:00000010:3.1:1788571896.602588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e400.
00010000:00000010:3.1:1788571896.602592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48a00.
00010000:00000010:3.1:1788571896.602595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49e00.
00010000:00000010:3.1:1788571896.602598:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384800.
00010000:00000010:3.1:1788571896.602601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e200.
00010000:00000010:3.1:1788571896.602604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d3ced00.
00010000:00000010:3.1:1788571896.602606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163e000.
00010000:00000010:3.1:1788571896.602610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343deb400.
00010000:00000010:3.1:1788571896.602613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341519400.
00010000:00000010:3.1:1788571896.602618:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343deb600.
00010000:00000010:3.1:1788571896.602621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341518a00.
00010000:00000010:3.1:1788571896.602624:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343deba00.
00010000:00000010:3.1:1788571896.602627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341518800.
00010000:00000010:3.1:1788571896.602630:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343deb000.
00010000:00000010:3.1:1788571896.602633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669000.
00010000:00000010:3.1:1788571896.602704:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343deb800.
00010000:00000010:3.1:1788571896.602708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668200.
00010000:00000010:3.1:1788571896.602711:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48e00.
00010000:00000010:3.1:1788571896.602714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0ca800.
00010000:00000010:3.1:1788571896.602717:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343debd00.
00010000:00000010:3.1:1788571896.602720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669800.
00010000:00000010:3.1:1788571896.602723:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384c00.
00010000:00000010:3.1:1788571896.602725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383ee00.
00010000:00000010:3.1:1788571896.602728:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7300.
00010000:00000010:3.1:1788571896.602731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66800.
00010000:00000010:3.1:1788571896.602734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384000.
00010000:00000010:3.1:1788571896.602737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f400.
00010000:00000010:3.1:1788571896.602741:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534824de00.
00010000:00000010:3.1:1788571896.602744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668e00.
00010000:00000010:3.1:1788571896.602748:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2400.
00010000:00000010:3.1:1788571896.602751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383ec00.
00010000:00000010:3.1:1788571896.602754:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2700.
00010000:00000010:3.1:1788571896.602757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a000.
00010000:00000010:3.1:1788571896.602760:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895351eb2000.
00010000:00000010:3.1:1788571896.602764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b600.
00010000:00000010:3.1:1788571896.602766:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534824d100.
00010000:00000010:3.1:1788571896.602769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668400.
00010000:00000010:3.1:1788571896.602773:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3500.
00010000:00000010:3.1:1788571896.602776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b400.
00010000:00000010:3.1:1788571896.602779:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534824df00.
00010000:00000010:3.1:1788571896.602781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668000.
00010000:00000010:3.1:1788571896.602786:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7af00.
00010000:00000010:3.1:1788571896.602789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dfa00.
00010000:00000010:3.1:1788571896.602792:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7a00.
00010000:00000010:3.1:1788571896.602795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67200.
00010000:00000010:3.1:1788571896.602799:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953489cbd00.
00010000:00000010:3.1:1788571896.602803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56200.
00010000:00000010:3.1:1788571896.602806:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953489cb400.
00010000:00000010:3.1:1788571896.602808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56e00.
00010000:00000010:3.1:1788571896.602881:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a400.
00010000:00000010:3.1:1788571896.602884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654dfc00.
00010000:00000010:3.1:1788571896.602887:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a100.
00010000:00000010:3.1:1788571896.602890:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654de400.
00010000:00000010:3.1:1788571896.602893:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953489cb500.
00010000:00000010:3.1:1788571896.602896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56400.
00010000:00000010:3.1:1788571896.602898:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953489cb300.
00010000:00000010:3.1:1788571896.602902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57e00.
00010000:00000010:3.1:1788571896.602905:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b4de200.
00010000:00000010:3.1:1788571896.602908:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57000.
00010000:00000010:3.1:1788571896.602910:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7e00.
00010000:00000010:3.1:1788571896.602913:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67800.
00010000:00000010:3.1:1788571896.602916:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbbc00.
00010000:00000010:3.1:1788571896.602919:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895382162e00.
00010000:00000010:3.1:1788571896.602923:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ab00.
00010000:00000010:3.1:1788571896.602927:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0400.
00010000:00000010:3.1:1788571896.602931:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a900.
00010000:00000010:3.1:1788571896.602934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57400.
00010000:00000010:3.1:1788571896.602936:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a000.
00010000:00000010:3.1:1788571896.602939:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56800.
00010000:00000010:3.1:1788571896.602942:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545ad00.
00010000:00000010:3.1:1788571896.602945:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57200.
00010000:00000010:3.1:1788571896.602947:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a400.
00010000:00000010:3.1:1788571896.602950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2aa00.
00010000:00000010:3.1:1788571896.602953:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545ab00.
00010000:00000010:3.1:1788571896.602955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2a000.
00010000:00000010:1.1:1788571896.665487:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534a59da00.
00010000:00000010:1.1:1788571896.665499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:1.1:1788571896.665504:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412aba00.
00010000:00000010:1.1:1788571896.665507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:1.1:1788571896.665510:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953602ef100.
00010000:00000010:1.1:1788571896.665513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00010000:00000010:1.1:1788571896.665517:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d0bb600.
00010000:00000010:1.1:1788571896.665519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:1.1:1788571896.665522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325300.
00010000:00000010:1.1:1788571896.665526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947c200.
00010000:00000010:1.1:1788571896.665528:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d0bba00.
00010000:00000010:1.1:1788571896.665530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:1.1:1788571896.665533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953602eff00.
00010000:00000010:1.1:1788571896.665536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383ea00.
00010000:00000010:1.1:1788571896.665539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534b325d00.
00010000:00000010:1.1:1788571896.665542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947d400.
00010000:00000010:1.1:1788571896.665545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d0bb500.
00010000:00000010:1.1:1788571896.665548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163fc00.
00010000:00000010:1.1:1788571896.665552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7700.
00010000:00000010:1.1:1788571896.665555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66000.
00010000:00000010:1.1:1788571896.665558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d3cea00.
00010000:00000010:1.1:1788571896.665561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163f200.
00010000:00000010:1.1:1788571896.665565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48400.
00010000:00000010:1.1:1788571896.665568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49200.
00010000:00000010:1.1:1788571896.665572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d3ce000.
00010000:00000010:1.1:1788571896.665575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163e600.
00010000:00000010:1.1:1788571896.665577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48700.
00010000:00000010:1.1:1788571896.665580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48400.
00010000:00000010:1.1:1788571896.665582:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d3ce800.
00010000:00000010:1.1:1788571896.665586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535163fe00.
00010000:00000010:1.1:1788571896.665603:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7000.
00010000:00000010:1.1:1788571896.665607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67000.
00010000:00000010:1.1:1788571896.665611:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384500.
00010000:00000010:1.1:1788571896.665614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383fe00.
00010000:00000010:1.1:1788571896.665617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d3ce400.
00010000:00000010:1.1:1788571896.665620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341519200.
00010000:00000010:1.1:1788571896.665624:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48800.
00010000:00000010:1.1:1788571896.665627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00010000:00000010:1.1:1788571896.665629:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384b00.
00010000:00000010:1.1:1788571896.665632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383f000.
00010000:00000010:1.1:1788571896.665749:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343deb500.
00010000:00000010:1.1:1788571896.665755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668a00.
00010000:00000010:1.1:1788571896.665757:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343deb300.
00010000:00000010:1.1:1788571896.665760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669e00.
00010000:00000010:1.1:1788571896.665763:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48c00.
00010000:00000010:1.1:1788571896.665766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ba00.
00010000:00000010:1.1:1788571896.665769:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48000.
00010000:00000010:1.1:1788571896.665772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cba00.
00010000:00000010:1.1:1788571896.665774:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c48f00.
00010000:00000010:1.1:1788571896.665777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0ca200.
00010000:00000010:1.1:1788571896.665781:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a900.
00010000:00000010:1.1:1788571896.665784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d0cb200.
00010000:00000010:1.1:1788571896.665791:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384600.
00010000:00000010:1.1:1788571896.665794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e600.
00010000:00000010:1.1:1788571896.665797:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895352384900.
00010000:00000010:1.1:1788571896.665799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535383e800.
00010000:00000010:1.1:1788571896.665806:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534824db00.
00010000:00000010:1.1:1788571896.665809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668600.
00010000:00000010:1.1:1788571896.665893:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534824dc00.
00010000:00000010:1.1:1788571896.665896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668800.
00010000:00000010:1.1:1788571896.665898:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534824d200.
00010000:00000010:1.1:1788571896.665901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379668c00.
00010000:00000010:1.1:1788571896.665905:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3900.
00010000:00000010:1.1:1788571896.665908:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542a600.
00010000:00000010:1.1:1788571896.665910:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3e00.
00010000:00000010:1.1:1788571896.665913:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542b200.
00010000:00000010:1.1:1788571896.665917:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534824d900.
00010000:00000010:1.1:1788571896.665920:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669c00.
00010000:00000010:1.1:1788571896.665922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89534824d700.
00010000:00000010:1.1:1788571896.665925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895379669a00.
00010000:00000010:1.1:1788571896.665933:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a500.
00010000:00000010:1.1:1788571896.665937:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654df000.
00010000:00000010:1.1:1788571896.665940:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7aa00.
00010000:00000010:1.1:1788571896.665943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654de000.
00010000:00000010:1.1:1788571896.665946:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3800.
00010000:00000010:1.1:1788571896.665948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536542ac00.
00010000:00000010:1.1:1788571896.665956:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953489cb600.
00010000:00000010:1.1:1788571896.665960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56000.
00010000:00000010:1.1:1788571896.665963:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a000.
00010000:00000010:1.1:1788571896.665965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953654df600.
00010000:00000010:1.1:1788571896.665974:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953489cba00.
00010000:00000010:1.1:1788571896.665976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56c00.
00010000:00000010:1.1:1788571896.665980:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953489cb000.
00010000:00000010:1.1:1788571896.665983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57c00.
00010000:00000010:1.1:1788571896.665985:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953489cb800.
00010000:00000010:1.1:1788571896.665988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c57a00.
00010000:00000010:1.1:1788571896.665991:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348db7f00.
00010000:00000010:1.1:1788571896.665994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66200.
00010000:00000010:1.1:1788571896.665996:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342ae3000.
00010000:00000010:1.1:1788571896.665999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a000.
00010000:00000010:1.1:1788571896.666003:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a600.
00010000:00000010:1.1:1788571896.666006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c56600.
00010000:00000010:1.1:1788571896.666014:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155300.
00010000:00000010:1.1:1788571896.666018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe1800.
00010000:00000010:1.1:1788571896.666022:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8a00.
00010000:00000010:1.1:1788571896.666025:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165a200.
00010000:00000010:1.1:1788571896.666028:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895345fbbb00.
00010000:00000010:1.1:1788571896.666032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895365434200.
00010000:00000010:1.1:1788571896.666039:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953454d8100.
00010000:00000010:1.1:1788571896.666042:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89535165ba00.
00010000:00000010:1.1:1788571896.666045:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895382155900.
00010000:00000010:1.1:1788571896.666048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895350fe0600.
00010000:00000010:1.1:1788571896.666050:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536545a300.
00010000:00000010:1.1:1788571896.666053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f2bc00.
00010000:00000010:1.1:1788571896.666057:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7a800.
00010000:00000010:1.1:1788571896.666060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c67400.
00010000:00000010:1.1:1788571896.666062:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536cc7a200.
00010000:00000010:1.1:1788571896.666065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c66400.
00010000:00000010:3.1:1788571903.304470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788571903.330463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895350fafb80.
00010000:00000010:0.1:1788571903.407758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:3.1:1788571915.969127:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56300.
00000080:00000010:0.1:1788571916.147756:0:179193:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc00.
00000080:00000010:0.1:1788571916.147773:0:179193:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5320.
00000080:00000010:0.1:1788571916.147778:0:179193:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb680.
00000080:00000010:0.1:1788571916.147782:0:179193:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1e0.
00010000:00000010:0.1:1788571922.439687:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4d00.
00010000:00000010:0.1:1788571922.439704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98e00.
00000080:00000010:0.1:1788571953.142483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00010000:00000010:0.1:1788571953.142500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:0.1:1788571953.142507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:3.1:1788572232.664443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00010000:00000010:1.1:1788572233.027327:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572233.027339:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00010000:00000010:1.1:1788572233.027343:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572233.046432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f80.
00000080:00000010:1.1:1788572233.046461:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463a6300.
00010000:00000010:1.1:1788572233.046472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572233.046475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572233.053628:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c52e0.
00000080:00000010:1.1:1788572233.053656:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535192eb40.
00010000:00000010:0.1:1788572233.062474:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572233.062493:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:3.1:1788572233.122602:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572233.122617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572233.198521:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349540000.
00010000:00000010:3.1:1788572233.198756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:3.1:1788572233.198762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:3.1:1788572233.198766:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5780.
00000080:00000010:3.1:1788572233.198775:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534634dac0.
00010000:00000010:3.1:1788572233.209568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572233.209584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572233.224470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8100.
00000080:00000010:3.1:1788572233.224502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353234200.
00010000:00000010:3.1:1788572233.239741:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572233.239757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572233.250572:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8e60.
00000080:00000010:3.1:1788572233.250591:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3e300.
00010000:00000010:0.1:1788572233.295457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572233.295465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572233.295472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd40.
00000080:00000010:0.1:1788572233.295487:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b518c0.
00010000:00000010:0.1:1788572233.301467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572233.301486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572233.326730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b20.
00000080:00000010:0.1:1788572233.326785:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497418c0.
00000080:00000010:3.1:1788572233.399211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb800.
00000080:00000010:3.1:1788572233.399242:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533d2940.
00010000:00000010:0.1:1788572233.419497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:0.1:1788572233.419518:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:0.1:1788572233.431363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8e60.
00000080:00000010:0.1:1788572233.431404:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b5e300.
00010000:00000010:1.1:1788572233.537121:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572233.537128:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572233.547635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc80.
00000080:00000010:1.1:1788572233.547667:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352699080.
00010000:00000010:1.1:1788572233.558880:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572233.558896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572233.582521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58a0.
00000080:00000010:1.1:1788572233.582555:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351975280.
00010000:00000010:0.1:1788572233.587178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:0.1:1788572233.587200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00010000:00000010:0.1:1788572233.640283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572233.640297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572233.649236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc00.
00000080:00000010:0.1:1788572233.656579:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d39c0.
00010000:00000010:0.1:1788572233.661670:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572233.661680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572233.665885:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c60.
00000080:00000010:0.1:1788572233.665920:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d0840.
00010000:00000010:0.1:1788572233.686697:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572233.686716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572233.707203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa50e0.
00000080:00000010:1.1:1788572233.707234:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518d2940.
00010000:00000010:3.1:1788572233.821512:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572233.821529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572233.826949:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb080.
00000080:00000010:3.1:1788572233.832533:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422318c0.
00010000:00000010:3.1:1788572233.838450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572233.838467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572233.852435:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572233.852454:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421aa940.
00010000:00000010:3.1:1788572233.858468:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572233.858485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572233.879457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8640.
00000080:00000010:3.1:1788572233.879484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b5ca40.
00010000:00000010:3.1:1788572233.887677:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572233.887695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00010000:00000010:3.1:1788572233.970709:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572233.970724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572233.979477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb480.
00000080:00000010:3.1:1788572233.979507:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354474a40.
00010000:00000010:3.1:1788572233.995084:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572233.995103:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572234.007526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8640.
00000080:00000010:3.1:1788572234.011467:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354475ac0.
00000080:00000010:0.1:1788572234.088308:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb940.
00000080:00000010:0.1:1788572234.088332:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353266300.
00010000:00000010:0.1:1788572234.094469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572234.094486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572234.161873:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb20.
00000080:00000010:0.1:1788572234.161889:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495db180.
00000080:00000010:3.1:1788572234.276739:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351926300.
00010000:00000010:3.1:1788572234.279942:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572234.279956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572234.284680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8640.
00000080:00000010:3.1:1788572234.284707:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534631a940.
00010000:00000010:3.1:1788572234.292363:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572234.292390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572234.312221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8140.
00000080:00000010:3.1:1788572234.312243:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463a4a40.
00010000:00000010:0.1:1788572234.532887:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:0.1:1788572234.532904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572234.594648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb60.
00000080:00000010:3.1:1788572234.594681:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352704200.
00010000:00000010:3.1:1788572234.621855:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572234.621865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572234.621869:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8140.
00000080:00000010:3.1:1788572234.621884:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535263a940.
00010000:00000010:3.1:1788572234.638598:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572234.638606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:3.1:1788572234.733135:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572234.733152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572234.758439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb140.
00000080:00000010:3.1:1788572234.758470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354438840.
00010000:00000010:3.1:1788572234.767557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572234.767571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572234.784485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8140.
00000080:00000010:3.1:1788572234.784511:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349664a40.
00010000:00000010:3.1:1788572234.788436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572234.788451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572234.814943:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f40.
00000080:00000010:3.1:1788572234.814962:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342308000.
00010000:00000010:3.1:1788572234.823134:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572234.823145:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:3.1:1788572235.020416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:3.1:1788572235.020434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572235.117422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0a0.
00000080:00000010:3.1:1788572235.117437:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354518000.
00010000:00000010:0.1:1788572235.130516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:0.1:1788572235.130528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:0.1:1788572235.153385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb740.
00000080:00000010:0.1:1788572235.153424:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b7e300.
00000080:00000010:0.1:1788572235.243661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb420.
00000080:00000010:0.1:1788572235.243688:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421bdac0.
00010000:00000010:0.1:1788572235.251419:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572235.251437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572235.270564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5d60.
00000080:00000010:0.1:1788572235.270595:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b239c0.
00010000:00000010:0.1:1788572235.299695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572235.299715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572235.334463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a00.
00000080:00000010:0.1:1788572235.334493:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352588000.
00010000:00000010:0.1:1788572235.361536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572235.361558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00010000:00000010:3.1:1788572235.437557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572235.437577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572235.453919:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd00.
00000080:00000010:3.1:1788572235.453949:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354535ac0.
00010000:00000010:3.1:1788572235.478603:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572235.478619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572235.482330:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00000080:00000010:3.1:1788572235.482361:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351910840.
00010000:00000010:3.1:1788572235.490814:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572235.490831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572235.497522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8880.
00000080:00000010:3.1:1788572235.497547:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422f8000.
00010000:00000010:3.1:1788572235.622539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:3.1:1788572235.622555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572235.793915:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5e0.
00000080:00000010:3.1:1788572235.793933:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349445280.
00010000:00000010:3.1:1788572235.797756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572235.797772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572235.812197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8880.
00000080:00000010:3.1:1788572235.812226:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342178000.
00010000:00000010:3.1:1788572235.947202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572235.947218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572235.951558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb620.
00000080:00000010:3.1:1788572235.951587:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a6b40.
00010000:00000010:0.1:1788572236.071235:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:0.1:1788572236.071255:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:0.1:1788572236.138237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8c0.
00000080:00000010:0.1:1788572236.138265:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533e8840.
00000080:00000010:3.1:1788572236.189709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb640.
00000080:00000010:3.1:1788572236.189733:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351abeb40.
00010000:00000010:3.1:1788572236.201484:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572236.201500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572236.207801:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d89e0.
00000080:00000010:3.1:1788572236.219447:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546fb180.
00010000:00000010:3.1:1788572236.238468:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572236.238485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572236.249712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:3.1:1788572236.253933:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352692100.
00010000:00000010:3.1:1788572236.259485:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572236.259500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572236.278469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d89e0.
00000080:00000010:3.1:1788572236.278498:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354530000.
00010000:00000010:3.1:1788572236.402475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:3.1:1788572236.402501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572236.416708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb680.
00000080:00000010:3.1:1788572236.416734:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf6b40.
00010000:00000010:3.1:1788572236.431787:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572236.431808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572236.473572:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d89e0.
00000080:00000010:3.1:1788572236.473626:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b84a40.
00010000:00000010:3.1:1788572236.480172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572236.480180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572236.484219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fa0.
00000080:00000010:3.1:1788572236.484250:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352746300.
00010000:00000010:3.1:1788572236.497756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572236.497775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:0.1:1788572236.563730:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:0.1:1788572236.563739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:1.1:1788572236.657742:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352757380.
00010000:00000010:3.1:1788572236.662457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572236.662473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572236.662479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:3.1:1788572236.767442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfc0.
00000080:00000010:3.1:1788572236.767468:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349759080.
00010000:00000010:3.1:1788572236.771771:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572236.771784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572236.776444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fa0.
00000080:00000010:3.1:1788572236.776473:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421ef380.
00010000:00000010:0.1:1788572236.799462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:0.1:1788572236.799480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:1.1:1788572236.821445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d85a0.
00000080:00000010:1.1:1788572236.821474:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b46300.
00010000:00000010:1.1:1788572236.842470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572236.842485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572236.850939:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53e0.
00000080:00000010:1.1:1788572236.850969:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534218c200.
00010000:00000010:1.1:1788572236.866671:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572236.866688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572236.876949:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5620.
00000080:00000010:1.1:1788572236.876975:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346334200.
00000080:00000010:3.1:1788572236.988787:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf80.
00000080:00000010:3.1:1788572236.988811:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352568000.
00010000:00000010:3.1:1788572236.996432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572236.996503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572237.027431:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fa0.
00000080:00000010:3.1:1788572237.027511:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533798c0.
00010000:00000010:3.1:1788572237.040108:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572237.040122:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572237.051442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8280.
00000080:00000010:3.1:1788572237.051473:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351af6300.
00010000:00000010:3.1:1788572237.077170:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572237.077189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572237.085460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fa0.
00000080:00000010:3.1:1788572237.085489:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346220000.
00000080:00000010:0.1:1788572237.128839:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3e0.
00000080:00000010:0.1:1788572237.132434:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526139c0.
00010000:00000010:0.1:1788572237.153588:0:2359:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572237.153597:0:2359:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00010000:00000010:0.1:1788572237.215657:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572237.215671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:0.1:1788572237.226292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8e0.
00000080:00000010:0.1:1788572237.226321:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535256a940.
00000080:00000010:3.1:1788572237.260846:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd60.
00000080:00000010:3.1:1788572237.260873:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a63180.
00010000:00000010:3.1:1788572237.271804:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572237.271831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572237.285907:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fa0.
00000080:00000010:3.1:1788572237.289435:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a62940.
00010000:00000010:1.1:1788572237.312591:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:1.1:1788572237.312602:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:0.1:1788572237.385751:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb300.
00000080:00000010:0.1:1788572237.385769:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622a100.
00010000:00000010:1.1:1788572237.391438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572237.391533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:1.1:1788572237.400469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5320.
00000080:00000010:1.1:1788572237.400502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346030840.
00010000:00000010:1.1:1788572237.410437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572237.410466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572237.547430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba40.
00000080:00000010:1.1:1788572237.547526:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423339c0.
00000080:00000010:3.1:1788572237.571143:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb000.
00000080:00000010:3.1:1788572237.571164:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535452dac0.
00010000:00000010:3.1:1788572237.578872:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572237.578888:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572237.587533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fa0.
00000080:00000010:3.1:1788572237.587562:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535181b9c0.
00010000:00000010:3.1:1788572237.645463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:3.1:1788572237.645479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572237.655461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:3.1:1788572237.655491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351938000.
00010000:00000010:3.1:1788572237.663468:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572237.663482:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00010000:00000010:3.1:1788572237.753442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572237.753461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572237.769639:0:31:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe00.
00000080:00000010:3.1:1788572237.780452:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534975f380.
00010000:00000010:3.1:1788572237.895454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:3.1:1788572237.895474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572237.904586:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:3.1:1788572237.909508:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ae4a40.
00010000:00000010:3.1:1788572237.922534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572237.922542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572237.931437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572237.931672:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534600a940.
00010000:00000010:3.1:1788572238.007667:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572238.007675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572238.040811:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00000080:00000010:3.1:1788572238.043466:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b04200.
00010000:00000010:1.1:1788572238.091532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572238.091544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:1.1:1788572238.184760:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb560.
00000080:00000010:1.1:1788572238.184783:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b5b9c0.
00010000:00000010:1.1:1788572238.216432:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572238.216439:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572238.216609:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5800.
00000080:00000010:1.1:1788572238.216623:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534944dac0.
00010000:00000010:1.1:1788572238.216632:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572238.216635:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572238.221745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53c0.
00000080:00000010:1.1:1788572238.224516:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352639080.
00010000:00000010:1.1:1788572238.232435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572238.232451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:0.1:1788572238.241116:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5800.
00000080:00000010:0.1:1788572238.242613:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534638ca40.
00010000:00000010:0.1:1788572238.252765:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572238.252771:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572238.265838:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5920.
00000080:00000010:0.1:1788572238.265871:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534215f380.
00010000:00000010:0.1:1788572238.269770:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572238.269777:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572238.278485:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5420.
00000080:00000010:0.1:1788572238.278515:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d6b40.
00010000:00000010:0.1:1788572238.385010:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:0.1:1788572238.385018:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:0.1:1788572238.396443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1c0.
00000080:00000010:0.1:1788572238.396469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620b180.
00010000:00000010:0.1:1788572238.396479:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572238.396484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572238.405470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa58a0.
00000080:00000010:0.1:1788572238.405570:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534212b180.
00010000:00000010:1.1:1788572238.420650:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:1.1:1788572238.420661:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572238.431418:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5060.
00000080:00000010:0.1:1788572238.431435:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a85ac0.
00010000:00000010:0.1:1788572238.446343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572238.446360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572238.451085:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa58a0.
00000080:00000010:0.1:1788572238.451116:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba2940.
00010000:00000010:0.1:1788572238.467784:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572238.467792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572238.484439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5060.
00000080:00000010:0.1:1788572238.484483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346304a40.
00010000:00000010:0.1:1788572238.484497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572238.484502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572238.489741:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa58a0.
00000080:00000010:0.1:1788572238.489757:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527a5ac0.
00010000:00000010:3.1:1788572238.519768:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572238.519777:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:0.1:1788572238.635453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572238.635467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:0.1:1788572238.644439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb160.
00000080:00000010:0.1:1788572238.644464:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544e5ac0.
00010000:00000010:0.1:1788572238.666729:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572238.666938:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00010000:00000010:0.1:1788572238.753676:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:0.1:1788572238.753686:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:0.1:1788572238.770663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfa0.
00000080:00000010:0.1:1788572238.770683:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349560840.
00010000:00000010:0.1:1788572238.782880:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572238.782886:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572238.792074:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5160.
00000080:00000010:0.1:1788572238.792095:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b09080.
00010000:00000010:0.1:1788572238.792104:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572238.792109:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572238.797021:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e80.
00000080:00000010:0.1:1788572238.803505:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533c4200.
00010000:00000010:1.1:1788572238.821666:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:1.1:1788572238.821682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:3.1:1788572238.851719:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3a0.
00000080:00000010:3.1:1788572238.851740:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422fb9c0.
00010000:00000010:3.1:1788572238.878128:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572238.878139:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00010000:00000010:0.1:1788572238.888741:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572238.888763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572238.892163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572238.892181:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463bca40.
00010000:00000010:3.1:1788572239.026159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572239.026175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572239.031721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb180.
00000080:00000010:3.1:1788572239.031743:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342316300.
00010000:00000010:1.1:1788572239.039474:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572239.039492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:1.1:1788572239.049557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:1.1:1788572239.049589:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349528000.
00010000:00000010:1.1:1788572239.068451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572239.068461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572239.072923:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50a0.
00000080:00000010:1.1:1788572239.072942:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953545018c0.
00010000:00000010:1.1:1788572239.097749:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572239.097762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572239.103367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5120.
00000080:00000010:1.1:1788572239.103388:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354409080.
00010000:00000010:1.1:1788572239.108435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572239.108449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572239.124666:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50a0.
00000080:00000010:1.1:1788572239.124692:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a6a100.
00010000:00000010:0.1:1788572239.144009:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:0.1:1788572239.144024:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:0.1:1788572239.226307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbaa0.
00000080:00000010:0.1:1788572239.229700:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534215b9c0.
00010000:00000010:0.1:1788572239.239492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572239.239514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572239.261191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c00.
00000080:00000010:0.1:1788572239.261218:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421a8000.
00010000:00000010:0.1:1788572239.267659:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572239.267668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572239.272463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5260.
00000080:00000010:0.1:1788572239.272488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b52940.
00010000:00000010:0.1:1788572239.332450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572239.332468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572239.332474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4a0.
00000080:00000010:0.1:1788572239.336908:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535448b9c0.
00010000:00000010:0.1:1788572239.353940:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572239.353955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:0.1:1788572239.367590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a40.
00000080:00000010:0.1:1788572239.367620:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349755ac0.
00010000:00000010:0.1:1788572239.379760:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572239.379776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:0.1:1788572239.390449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5140.
00000080:00000010:0.1:1788572239.395543:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342234200.
00010000:00000010:1.1:1788572239.411003:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572239.411013:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:0.1:1788572239.414550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a40.
00000080:00000010:0.1:1788572239.414567:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421d4a40.
00000080:00000010:3.1:1788572239.520708:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:3.1:1788572239.520725:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535447c200.
00000080:00000010:0.1:1788572239.597752:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:0.1:1788572239.597768:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535475e300.
00010000:00000010:1.1:1788572239.600406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572239.600423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00010000:00000010:1.1:1788572239.607616:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572239.607640:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:1.1:1788572239.607647:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5800.
00000080:00000010:1.1:1788572239.607664:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535326c200.
00010000:00000010:1.1:1788572239.626560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572239.626575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572239.633659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5020.
00000080:00000010:1.1:1788572239.633678:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a39c0.
00010000:00000010:1.1:1788572239.633691:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572239.633696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572239.639826:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5840.
00000080:00000010:1.1:1788572239.639848:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533db9c0.
00010000:00000010:1.1:1788572239.646297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572239.646311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572239.654570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00000080:00000010:1.1:1788572239.654590:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534632eb40.
00010000:00000010:1.1:1788572239.672600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572239.672607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:3.1:1788572239.694611:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:3.1:1788572239.694655:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b48840.
00010000:00000010:3.1:1788572239.694662:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572239.694667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572239.704554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fc0.
00000080:00000010:3.1:1788572239.706613:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534633d280.
00010000:00000010:3.1:1788572239.716619:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572239.716688:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572239.734551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8cc0.
00000080:00000010:3.1:1788572239.734577:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535448a940.
00010000:00000010:3.1:1788572239.775838:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572239.775855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572239.782608:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb960.
00000080:00000010:3.1:1788572239.782627:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351970840.
00010000:00000010:3.1:1788572239.782699:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572239.782704:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572239.789569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8cc0.
00000080:00000010:3.1:1788572239.789589:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535199b180.
00000080:00000010:3.1:1788572239.848653:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6a0.
00000080:00000010:3.1:1788572239.848677:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d0000.
00010000:00000010:3.1:1788572239.853265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572239.853280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572239.871456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8cc0.
00000080:00000010:3.1:1788572239.871482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421d5ac0.
00010000:00000010:3.1:1788572239.879834:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572239.879849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572239.880431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d85a0.
00000080:00000010:3.1:1788572239.880492:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535198ca40.
00000080:00000010:0.1:1788572239.937926:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8cc0.
00000080:00000010:0.1:1788572239.937980:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535327b9c0.
00010000:00000010:1.1:1788572239.962427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:1.1:1788572239.962434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572239.975551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:0.1:1788572239.975579:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535447a100.
00010000:00000010:3.1:1788572240.032696:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572240.032703:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572240.045329:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbe0.
00000080:00000010:3.1:1788572240.045347:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463edac0.
00010000:00000010:3.1:1788572240.049436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572240.049582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572240.056098:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d85a0.
00000080:00000010:3.1:1788572240.056121:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535192d280.
00000080:00000010:3.1:1788572240.125051:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf40.
00000080:00000010:3.1:1788572240.125062:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349542100.
00000080:00000010:1.1:1788572240.134555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:1.1:1788572240.134572:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a39080.
00010000:00000010:1.1:1788572240.143434:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572240.143448:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572240.147761:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5720.
00000080:00000010:1.1:1788572240.147784:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346028000.
00010000:00000010:1.1:1788572240.151635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572240.151644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00000080:00000010:1.1:1788572240.156257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5340.
00000080:00000010:1.1:1788572240.156281:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532318c0.
00010000:00000010:1.1:1788572240.163734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572240.163750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572240.175486:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5720.
00000080:00000010:1.1:1788572240.175511:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535440f380.
00010000:00000010:1.1:1788572240.182087:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572240.182101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572240.200634:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5340.
00000080:00000010:1.1:1788572240.200654:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535326b9c0.
00010000:00000010:1.1:1788572240.204721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572240.204735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572240.219838:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5720.
00000080:00000010:1.1:1788572240.219856:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544eb180.
00010000:00000010:3.1:1788572240.339350:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572240.339359:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:0.1:1788572240.347927:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:0.1:1788572240.347947:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bd5ac0.
00010000:00000010:0.1:1788572240.361182:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572240.361193:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572240.366502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c80.
00000080:00000010:0.1:1788572240.366530:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534217d280.
00010000:00000010:1.1:1788572240.369963:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572240.369976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572240.374987:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5200.
00000080:00000010:0.1:1788572240.375016:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d6300.
00010000:00000010:0.1:1788572240.389353:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572240.389368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572240.402513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c80.
00000080:00000010:0.1:1788572240.408442:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346205280.
00000080:00000010:3.1:1788572240.453691:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5200.
00000080:00000010:3.1:1788572240.453720:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351abb9c0.
00010000:00000010:3.1:1788572240.463447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572240.463462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572240.472425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:3.1:1788572240.476450:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ab2940.
00010000:00000010:3.1:1788572240.487532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572240.487549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572240.510441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8880.
00000080:00000010:3.1:1788572240.516788:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a50000.
00010000:00000010:3.1:1788572240.578332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572240.578348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572240.585992:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8a0.
00000080:00000010:3.1:1788572240.589575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533f4200.
00010000:00000010:3.1:1788572240.593607:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572240.593620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572240.598733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8880.
00000080:00000010:3.1:1788572240.606694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494ec200.
00010000:00000010:0.1:1788572240.610631:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:0.1:1788572240.610644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:1.1:1788572240.626516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8a80.
00000080:00000010:1.1:1788572240.626547:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342228000.
00010000:00000010:1.1:1788572240.626553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572240.626558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572240.633993:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5240.
00000080:00000010:1.1:1788572240.637609:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b15280.
00000080:00000010:3.1:1788572240.655570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5360.
00000080:00000010:3.1:1788572240.655594:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535337b180.
00010000:00000010:1.1:1788572240.656525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572240.656537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00010000:00000010:3.1:1788572240.661528:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572240.661541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572240.672455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8880.
00000080:00000010:3.1:1788572240.672484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346028840.
00010000:00000010:3.1:1788572240.681463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572240.681479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572240.692572:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00000080:00000010:3.1:1788572240.692598:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a4dac0.
00000080:00000010:1.1:1788572240.745538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:1.1:1788572240.745569:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346042100.
00010000:00000010:1.1:1788572240.756187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572240.756203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572240.765024:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5920.
00000080:00000010:1.1:1788572240.770595:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e5ac0.
00010000:00000010:1.1:1788572240.783597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572240.783613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572240.787432:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:1.1:1788572240.787463:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497aca40.
00010000:00000010:0.1:1788572240.936669:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572240.936685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:3.1:1788572241.096755:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:3.1:1788572241.096773:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b7f380.
00010000:00000010:3.1:1788572241.096781:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572241.096786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572241.104209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00000080:00000010:3.1:1788572241.104234:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aca100.
00010000:00000010:3.1:1788572241.107983:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572241.107998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572241.120584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f40.
00000080:00000010:3.1:1788572241.120627:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494fa100.
00010000:00000010:0.1:1788572241.144444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:0.1:1788572241.144462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:3.1:1788572241.216550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572241.216569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572241.241614:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:3.1:1788572241.241642:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354613180.
00010000:00000010:3.1:1788572241.260448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572241.260465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572241.294070:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f40.
00010000:00000010:0.1:1788572241.335554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572241.335568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572241.340645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5d20.
00000080:00000010:0.1:1788572241.344670:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d98c0.
00010000:00000010:0.1:1788572241.354434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572241.354449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00010000:00000010:1.1:1788572241.726576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:1.1:1788572241.726583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:0.1:1788572241.807455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbec0.
00000080:00000010:0.1:1788572241.807469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354485ac0.
00010000:00000010:0.1:1788572241.821168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572241.821180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572241.821186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5780.
00000080:00000010:0.1:1788572241.821201:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601eb40.
00010000:00000010:0.1:1788572241.846445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572241.846452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:0.1:1788572241.857913:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5dc0.
00000080:00000010:0.1:1788572241.861503:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421eeb40.
00010000:00000010:0.1:1788572241.866037:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:0.1:1788572241.866063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572241.870436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5780.
00000080:00000010:0.1:1788572241.870469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351af4200.
00010000:00000010:0.1:1788572241.897446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572241.897460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:3.1:1788572241.942877:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:3.1:1788572241.942898:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421339c0.
00010000:00000010:3.1:1788572241.942910:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572241.942916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572241.976012:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f40.
00000080:00000010:3.1:1788572241.976039:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a7ca40.
00010000:00000010:3.1:1788572241.987873:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572241.987887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:0.1:1788572241.992567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8140.
00000080:00000010:0.1:1788572241.992597:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a6b180.
00010000:00000010:1.1:1788572242.004433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:1.1:1788572242.004447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:3.1:1788572242.046176:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb920.
00000080:00000010:3.1:1788572242.054092:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546c8000.
00010000:00000010:0.1:1788572242.057895:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:0.1:1788572242.057906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:0.1:1788572242.079723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:0.1:1788572242.079744:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b93180.
00010000:00000010:0.1:1788572242.086532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572242.086548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572242.094462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5de0.
00000080:00000010:0.1:1788572242.097575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346033180.
00010000:00000010:0.1:1788572242.102575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572242.102593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572242.111901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ae0.
00000080:00000010:0.1:1788572242.120694:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aa6300.
00010000:00000010:0.1:1788572242.131616:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572242.131633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572242.141435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5de0.
00000080:00000010:0.1:1788572242.141471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535330a100.
00010000:00000010:1.1:1788572242.145508:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572242.145522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572242.162435:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5860.
00000080:00000010:1.1:1788572242.162792:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526a98c0.
00010000:00000010:1.1:1788572242.167331:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572242.167348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572242.186257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5120.
00000080:00000010:1.1:1788572242.186277:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352610840.
00010000:00000010:0.1:1788572242.204747:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:0.1:1788572242.204755:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:3.1:1788572242.271673:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba80.
00000080:00000010:3.1:1788572242.271690:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534217a940.
00010000:00000010:3.1:1788572242.279352:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572242.279361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572242.292432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d81e0.
00000080:00000010:3.1:1788572242.292459:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535327e300.
00010000:00000010:3.1:1788572242.364862:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572242.364871:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572242.373180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:3.1:1788572242.373207:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354520000.
00010000:00000010:1.1:1788572242.413687:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:1.1:1788572242.413694:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572242.418726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0c0.
00000080:00000010:0.1:1788572242.418743:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518f6300.
00010000:00000010:1.1:1788572242.433442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572242.433457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:1.1:1788572242.445677:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5720.
00000080:00000010:1.1:1788572242.445693:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bba100.
00010000:00000010:1.1:1788572242.456526:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572242.456537:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572242.461947:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00000080:00000010:1.1:1788572242.461965:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354548840.
00010000:00000010:1.1:1788572242.468589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572242.468616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00010000:00000010:0.1:1788572242.503305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572242.503321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:0.1:1788572242.507580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb880.
00000080:00000010:0.1:1788572242.507598:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346010000.
00010000:00000010:0.1:1788572242.513739:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572242.513761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572242.516616:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5500.
00000080:00000010:0.1:1788572242.516636:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346218840.
00010000:00000010:0.1:1788572242.534469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572242.534481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572242.534486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56a0.
00000080:00000010:0.1:1788572242.534501:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497aeb40.
00010000:00000010:0.1:1788572242.539460:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572242.539477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00010000:00000010:3.1:1788572242.736453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:3.1:1788572242.736470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572242.736476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:3.1:1788572242.736491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349564200.
00010000:00000010:3.1:1788572242.751873:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572242.751888:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00010000:00000010:0.1:1788572243.012033:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572243.012048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:0.1:1788572243.019437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb060.
00000080:00000010:0.1:1788572243.023665:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346059080.
00010000:00000010:0.1:1788572243.038780:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572243.038788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572243.042863:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5820.
00000080:00000010:0.1:1788572243.042890:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a98840.
00000080:00000010:0.1:1788572243.121607:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:0.1:1788572243.121632:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a8a100.
00010000:00000010:0.1:1788572243.195456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572243.195472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:0.1:1788572243.206642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:0.1:1788572243.206670:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534234a100.
00010000:00000010:0.1:1788572243.229560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572243.229850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:3.1:1788572243.414569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:3.1:1788572243.414606:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349494200.
00010000:00000010:3.1:1788572243.423684:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572243.423698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572243.446491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572243.446514:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544e2940.
00010000:00000010:3.1:1788572243.540455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572243.540470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572243.552649:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:3.1:1788572243.552667:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be2940.
00010000:00000010:3.1:1788572243.562173:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572243.562182:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:0.1:1788572243.747050:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:0.1:1788572243.747077:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533e0000.
00010000:00000010:3.1:1788572243.782468:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572243.782483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572243.793584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:3.1:1788572243.793616:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534944eb40.
00000080:00000010:0.1:1788572243.853947:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:0.1:1788572243.853968:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342108840.
00010000:00000010:0.1:1788572243.860386:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572243.860415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572243.870018:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5340.
00000080:00000010:0.1:1788572243.870048:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346307380.
00010000:00000010:1.1:1788572243.902951:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572243.902959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:1.1:1788572243.910545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:1.1:1788572243.910577:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463de300.
00010000:00000010:1.1:1788572243.910583:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572243.910587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572243.918467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00000080:00000010:1.1:1788572243.918493:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535190b180.
00010000:00000010:1.1:1788572243.929675:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572243.929687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572243.935450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:1.1:1788572243.935479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ae8000.
00010000:00000010:1.1:1788572243.943452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572243.943467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572243.957259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00000080:00000010:1.1:1788572243.957287:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aeca40.
00010000:00000010:1.1:1788572243.961560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572243.961578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572243.975469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:1.1:1788572243.975485:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535180dac0.
00010000:00000010:3.1:1788572244.081798:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572244.081812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572244.089204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:3.1:1788572244.089239:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544a4a40.
00010000:00000010:0.1:1788572244.089427:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:0.1:1788572244.089437:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:0.1:1788572244.136832:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:0.1:1788572244.136864:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535190a940.
00010000:00000010:1.1:1788572244.148196:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572244.148206:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:1.1:1788572244.177723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:1.1:1788572244.180635:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354494a40.
00010000:00000010:1.1:1788572244.180657:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572244.180662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572244.188622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5600.
00000080:00000010:1.1:1788572244.188657:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535180b180.
00010000:00000010:1.1:1788572244.197570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572244.197585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572244.202199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d80.
00000080:00000010:1.1:1788572244.202230:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463be300.
00010000:00000010:1.1:1788572244.208704:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572244.208720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572244.226561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5600.
00000080:00000010:1.1:1788572244.226603:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354538840.
00010000:00000010:1.1:1788572244.226614:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572244.226619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572244.233499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d80.
00000080:00000010:1.1:1788572244.237912:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349658000.
00010000:00000010:1.1:1788572244.242690:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572244.242703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572244.256835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5600.
00000080:00000010:1.1:1788572244.256865:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e7380.
00010000:00000010:1.1:1788572244.260644:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572244.260652:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:0.1:1788572244.283490:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d80.
00000080:00000010:0.1:1788572244.283518:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b0f380.
00010000:00000010:0.1:1788572244.290725:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:0.1:1788572244.290736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00010000:00000010:3.1:1788572244.345235:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572244.345250:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572244.353042:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf20.
00000080:00000010:3.1:1788572244.353071:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a89080.
00010000:00000010:3.1:1788572244.365439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572244.365567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572244.371897:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d88c0.
00000080:00000010:3.1:1788572244.371916:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534212b9c0.
00000080:00000010:0.1:1788572244.436486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422de300.
00010000:00000010:3.1:1788572244.492574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572244.492586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572244.503787:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:3.1:1788572244.503811:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342345280.
00010000:00000010:3.1:1788572244.511602:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572244.511609:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572244.532371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d88c0.
00000080:00000010:3.1:1788572244.532408:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494918c0.
00010000:00000010:3.1:1788572244.541459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572244.541475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572244.541480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8980.
00000080:00000010:3.1:1788572244.541497:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba4200.
00010000:00000010:1.1:1788572244.756859:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:1.1:1788572244.756873:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00010000:00000010:3.1:1788572244.767934:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:3.1:1788572244.767951:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572244.795907:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf20.
00000080:00000010:0.1:1788572244.795934:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534210b180.
00010000:00000010:1.1:1788572245.029495:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572245.029511:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:1.1:1788572245.053546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:1.1:1788572245.053573:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349596300.
00010000:00000010:1.1:1788572245.086737:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572245.086763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00010000:00000010:0.1:1788572245.119534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:0.1:1788572245.119543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:0.1:1788572245.126568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:0.1:1788572245.126600:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342340840.
00010000:00000010:3.1:1788572245.256003:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572245.256018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572245.267791:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00010000:00000010:0.1:1788572245.291111:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:0.1:1788572245.291119:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:0.1:1788572245.300588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:0.1:1788572245.300616:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aeb9c0.
00010000:00000010:0.1:1788572245.312199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572245.312214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572245.325442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ce0.
00000080:00000010:0.1:1788572245.325467:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a15280.
00010000:00000010:1.1:1788572245.339657:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:1.1:1788572245.339677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572245.472507:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bc2100.
00010000:00000010:1.1:1788572245.492830:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:1.1:1788572245.492849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00010000:00000010:3.1:1788572245.553575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:3.1:1788572245.553599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572245.573443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:3.1:1788572245.577461:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354528840.
00010000:00000010:3.1:1788572245.631040:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572245.631056:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572245.653423:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:3.1:1788572245.653497:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b6ca40.
00010000:00000010:3.1:1788572245.805089:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572245.805097:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572245.818734:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:3.1:1788572245.818754:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601c200.
00010000:00000010:0.1:1788572245.866484:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572245.866491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:0.1:1788572245.884554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572245.884563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572245.890181:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:0.1:1788572245.890202:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534632b180.
00010000:00000010:0.1:1788572245.890211:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572245.890217:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572245.896430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa59a0.
00000080:00000010:0.1:1788572245.896452:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534952a100.
00010000:00000010:1.1:1788572245.904580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:1.1:1788572245.904592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572245.908533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5300.
00000080:00000010:0.1:1788572245.908552:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342313180.
00010000:00000010:0.1:1788572245.913437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572245.913443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572245.918455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5740.
00000080:00000010:0.1:1788572245.918470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a68840.
00010000:00000010:0.1:1788572245.927431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572245.927460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572245.932459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5300.
00000080:00000010:0.1:1788572245.932488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495fe300.
00010000:00000010:0.1:1788572245.935650:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572245.935658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572245.940415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5740.
00000080:00000010:0.1:1788572245.940654:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342130840.
00010000:00000010:1.1:1788572245.946443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:1.1:1788572245.946464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572246.167901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:0.1:1788572246.167926:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534606e300.
00010000:00000010:1.1:1788572246.176590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572246.176604:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:1.1:1788572246.186446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a20.
00000080:00000010:1.1:1788572246.186474:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e0000.
00000080:00000010:3.1:1788572246.285669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:3.1:1788572246.285693:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519b0000.
00010000:00000010:0.1:1788572246.288878:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:0.1:1788572246.288889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572246.336039:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:3.1:1788572246.336060:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535193ca40.
00000080:00000010:0.1:1788572246.356298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:0.1:1788572246.356326:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534967eb40.
00010000:00000010:0.1:1788572246.363865:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572246.363879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572246.379831:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b60.
00000080:00000010:0.1:1788572246.388752:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b2940.
00010000:00000010:0.1:1788572246.392963:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572246.392991:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00010000:00000010:3.1:1788572246.454595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:3.1:1788572246.454611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572246.490943:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:3.1:1788572246.490978:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b86300.
00010000:00000010:0.1:1788572246.502261:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:0.1:1788572246.502277:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00010000:00000010:0.1:1788572246.604531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:0.1:1788572246.604548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572246.625616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb060.
00000080:00000010:0.1:1788572246.625644:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953462139c0.
00010000:00000010:0.1:1788572246.659468:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:0.1:1788572246.659479:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572246.974612:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:3.1:1788572246.974642:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b28000.
00010000:00000010:3.1:1788572246.978680:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572246.978695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:0.1:1788572246.996425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c40.
00000080:00000010:0.1:1788572246.996460:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947a940.
00010000:00000010:1.1:1788572247.004658:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.004674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:1.1:1788572247.020337:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5760.
00000080:00000010:1.1:1788572247.020365:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534222d280.
00010000:00000010:0.1:1788572247.186664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:0.1:1788572247.186682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:0.1:1788572247.206435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0c0.
00000080:00000010:0.1:1788572247.206461:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b2a940.
00010000:00000010:0.1:1788572247.214625:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572247.214632:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572247.217546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5080.
00000080:00000010:0.1:1788572247.217570:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533c2100.
00010000:00000010:0.1:1788572247.222573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572247.222590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572247.232499:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5900.
00000080:00000010:0.1:1788572247.232520:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e6300.
00010000:00000010:3.1:1788572247.265534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572247.265548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572247.265555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:3.1:1788572247.265581:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535258ca40.
00010000:00000010:3.1:1788572247.274711:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572247.274726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572247.280688:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83a0.
00000080:00000010:3.1:1788572247.280848:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494e9080.
00010000:00000010:3.1:1788572247.300749:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572247.300758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:1.1:1788572247.401666:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.401673:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:1.1:1788572247.401678:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba80.
00000080:00000010:1.1:1788572247.401690:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534604e300.
00010000:00000010:1.1:1788572247.415156:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572247.415165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572247.415170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:1.1:1788572247.415184:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349444200.
00010000:00000010:1.1:1788572247.419516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.419528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572247.428345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50e0.
00000080:00000010:1.1:1788572247.428371:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346396300.
00010000:00000010:1.1:1788572247.437440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.437457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572247.449730:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:1.1:1788572247.449747:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352638000.
00010000:00000010:3.1:1788572247.551529:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572247.551546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:1.1:1788572247.763534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.763540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572247.770468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f40.
00000080:00000010:1.1:1788572247.770500:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354503180.
00010000:00000010:1.1:1788572247.779497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.779511:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572247.784554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5b20.
00000080:00000010:1.1:1788572247.784586:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535440a100.
00010000:00000010:1.1:1788572247.796577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.796589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572247.800732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f40.
00000080:00000010:1.1:1788572247.800751:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a52100.
00010000:00000010:1.1:1788572247.806105:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.806123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:0.1:1788572247.822232:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5b20.
00000080:00000010:0.1:1788572247.822262:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953545039c0.
00010000:00000010:1.1:1788572247.825467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.825481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:1.1:1788572247.840556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5220.
00000080:00000010:1.1:1788572247.840599:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420bd280.
00010000:00000010:1.1:1788572247.850087:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.850095:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572247.853609:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f20.
00000080:00000010:1.1:1788572247.853651:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3eb40.
00010000:00000010:1.1:1788572247.860706:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572247.860721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:0.1:1788572247.875667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d20.
00000080:00000010:0.1:1788572247.875693:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349547380.
00000080:00000010:3.1:1788572247.972449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb920.
00000080:00000010:3.1:1788572247.972478:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342181080.
00010000:00000010:3.1:1788572248.059459:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:3.1:1788572248.059474:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572248.072460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:3.1:1788572248.072489:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535326eb40.
00010000:00000010:3.1:1788572248.085452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572248.085466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572248.094222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d60.
00000080:00000010:3.1:1788572248.094242:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533aca40.
00010000:00000010:3.1:1788572248.106432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572248.106458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00010000:00000010:3.1:1788572248.129428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572248.129436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:0.1:1788572248.218691:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb880.
00000080:00000010:0.1:1788572248.218716:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354479080.
00010000:00000010:0.1:1788572248.226422:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572248.226428:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572248.235797:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53c0.
00000080:00000010:0.1:1788572248.235825:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535275f380.
00010000:00000010:1.1:1788572248.259449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:1.1:1788572248.259464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00010000:00000010:1.1:1788572248.292494:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572248.292513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:1.1:1788572248.308536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbec0.
00000080:00000010:1.1:1788572248.308567:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535275e300.
00010000:00000010:1.1:1788572248.324622:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572248.324635:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572248.332679:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5380.
00000080:00000010:1.1:1788572248.332697:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349753180.
00010000:00000010:1.1:1788572248.336565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572248.336581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:0.1:1788572248.347526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5620.
00000080:00000010:0.1:1788572248.357455:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534633c200.
00000080:00000010:3.1:1788572248.414781:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:3.1:1788572248.414808:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346320000.
00010000:00000010:3.1:1788572248.430817:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572248.430832:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572248.438421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d60.
00000080:00000010:3.1:1788572248.438450:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533da100.
00010000:00000010:3.1:1788572248.636973:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572248.636989:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572248.651985:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:3.1:1788572248.652017:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351976300.
00010000:00000010:3.1:1788572248.712441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572248.712458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572248.734756:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8500.
00000080:00000010:3.1:1788572248.734783:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353295ac0.
00010000:00000010:3.1:1788572248.741284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572248.741299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572248.749038:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:3.1:1788572248.749061:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353298840.
00010000:00000010:0.1:1788572248.825677:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:0.1:1788572248.825694:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:0.1:1788572248.841979:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572248.841996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:0.1:1788572248.849428:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:0.1:1788572248.849445:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620f380.
00010000:00000010:0.1:1788572248.963728:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:0.1:1788572248.963744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:0.1:1788572249.035584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.035594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572249.044601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8a0.
00000080:00000010:0.1:1788572249.048786:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421bd280.
00010000:00000010:0.1:1788572249.062439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.062454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572249.070601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5fc0.
00000080:00000010:0.1:1788572249.070618:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b34a40.
00010000:00000010:0.1:1788572249.075580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.075596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572249.087450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53e0.
00000080:00000010:0.1:1788572249.087480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534600ca40.
00010000:00000010:0.1:1788572249.100435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.100451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572249.107494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5fc0.
00000080:00000010:0.1:1788572249.107524:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342128000.
00010000:00000010:0.1:1788572249.115442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.115460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572249.124522:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53e0.
00000080:00000010:0.1:1788572249.124540:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532418c0.
00010000:00000010:0.1:1788572249.129579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.129597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572249.139680:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5fc0.
00000080:00000010:0.1:1788572249.139699:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a8e300.
00010000:00000010:0.1:1788572249.153675:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.153683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572249.158456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53e0.
00000080:00000010:0.1:1788572249.158483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230a940.
00010000:00000010:0.1:1788572249.166823:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.166833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00010000:00000010:0.1:1788572249.243806:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:0.1:1788572249.243824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572249.290829:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:0.1:1788572249.290845:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463dc200.
00010000:00000010:0.1:1788572249.299969:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.299976:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572249.309557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5840.
00000080:00000010:0.1:1788572249.309571:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534210d280.
00010000:00000010:1.1:1788572249.325533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572249.325647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00010000:00000010:1.1:1788572249.334433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572249.334451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:1.1:1788572249.339424:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:1.1:1788572249.339451:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532d6300.
00010000:00000010:1.1:1788572249.349822:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572249.349830:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572249.360191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ae0.
00000080:00000010:1.1:1788572249.360208:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b5dac0.
00010000:00000010:1.1:1788572249.367449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572249.367465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572249.383434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5340.
00000080:00000010:1.1:1788572249.383852:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534944a940.
00000080:00000010:3.1:1788572249.479472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:3.1:1788572249.479496:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546fd280.
00000080:00000010:1.1:1788572249.492498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:1.1:1788572249.492529:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544e39c0.
00010000:00000010:0.1:1788572249.506117:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:0.1:1788572249.506136:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:0.1:1788572249.510041:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5b20.
00000080:00000010:0.1:1788572249.510058:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342343180.
00010000:00000010:0.1:1788572249.515432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.515461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572249.527897:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5000.
00000080:00000010:0.1:1788572249.527913:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342085ac0.
00010000:00000010:0.1:1788572249.539951:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.539970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572249.539975:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a20.
00000080:00000010:0.1:1788572249.539991:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463c0840.
00010000:00000010:0.1:1788572249.551741:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.551758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572249.559472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5000.
00000080:00000010:0.1:1788572249.559500:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349635280.
00010000:00000010:0.1:1788572249.566634:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572249.566649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572249.573640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a20.
00000080:00000010:0.1:1788572249.573670:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349765ac0.
00010000:00000010:1.1:1788572249.596433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:1.1:1788572249.596440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00010000:00000010:3.1:1788572249.622139:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572249.622148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572249.628480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbe0.
00000080:00000010:1.1:1788572249.684454:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346300000.
00010000:00000010:1.1:1788572249.688521:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572249.688528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572249.692644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:1.1:1788572249.692670:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a14a40.
00010000:00000010:1.1:1788572249.697189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572249.697197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572249.701435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50e0.
00000080:00000010:1.1:1788572249.701459:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351908000.
00010000:00000010:1.1:1788572249.716289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572249.716300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572249.724525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:1.1:1788572249.724555:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354490000.
00010000:00000010:1.1:1788572249.739628:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572249.739635:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:3.1:1788572249.756503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:3.1:1788572249.756526:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544918c0.
00000080:00000010:1.1:1788572249.756653:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50e0.
00000080:00000010:1.1:1788572249.756663:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535451a940.
00010000:00000010:1.1:1788572249.756666:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572249.756670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00010000:00000010:0.1:1788572249.760428:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572249.760435:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:0.1:1788572249.790087:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:0.1:1788572249.790114:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422fdac0.
00000080:00000010:0.1:1788572249.831265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5420.
00000080:00000010:0.1:1788572249.831284:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474dac0.
00010000:00000010:3.1:1788572249.848437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572249.848456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572249.851661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf40.
00000080:00000010:3.1:1788572249.851691:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354558000.
00010000:00000010:3.1:1788572249.856675:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572249.856683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572249.860782:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87e0.
00000080:00000010:3.1:1788572249.860810:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349562100.
00010000:00000010:3.1:1788572249.896065:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572249.896083:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572249.903672:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6a0.
00000080:00000010:3.1:1788572249.903697:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535446a940.
00010000:00000010:3.1:1788572249.913873:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572249.913888:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:0.1:1788572249.924964:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87e0.
00000080:00000010:0.1:1788572249.924993:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953545218c0.
00000080:00000010:0.1:1788572250.067658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:0.1:1788572250.067677:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351985280.
00010000:00000010:0.1:1788572250.073584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572250.073601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572250.089865:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5460.
00000080:00000010:0.1:1788572250.089894:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b68840.
00010000:00000010:0.1:1788572250.094952:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572250.094960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572250.098823:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5fe0.
00000080:00000010:0.1:1788572250.098847:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf2940.
00010000:00000010:0.1:1788572250.115162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572250.115171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572250.122115:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5460.
00000080:00000010:0.1:1788572250.122140:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354542100.
00000080:00000010:0.1:1788572250.216778:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:0.1:1788572250.216806:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346217380.
00010000:00000010:1.1:1788572250.223750:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572250.223765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:1.1:1788572250.233691:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5aa0.
00000080:00000010:1.1:1788572250.233735:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534223ca40.
00010000:00000010:1.1:1788572250.244205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572250.244222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572250.251652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5920.
00000080:00000010:1.1:1788572250.251681:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534218eb40.
00010000:00000010:0.1:1788572250.264800:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572250.264810:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:0.1:1788572250.269456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00000080:00000010:0.1:1788572250.269484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b40840.
00010000:00000010:1.1:1788572250.279298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:1.1:1788572250.279313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00010000:00000010:3.1:1788572250.309471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:3.1:1788572250.309478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:0.1:1788572250.320738:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:0.1:1788572250.320748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572250.321010:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb960.
00000080:00000010:3.1:1788572250.321026:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534975a100.
00000080:00000010:0.1:1788572250.330233:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8580.
00000080:00000010:0.1:1788572250.330257:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352615ac0.
00010000:00000010:0.1:1788572250.337103:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572250.337117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572250.365439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5800.
00000080:00000010:0.1:1788572250.365457:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622ca40.
00010000:00000010:0.1:1788572250.365466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572250.365470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00010000:00000010:0.1:1788572250.384429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572250.384450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:1.1:1788572250.399025:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:1.1:1788572250.399055:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351af18c0.
00010000:00000010:1.1:1788572250.412771:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572250.412779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572250.416993:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c40.
00000080:00000010:1.1:1788572250.417019:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526a8000.
00010000:00000010:0.1:1788572250.427446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572250.427463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:0.1:1788572250.443854:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5320.
00000080:00000010:0.1:1788572250.443875:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aa6b40.
00010000:00000010:0.1:1788572250.450619:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572250.450637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572250.461036:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5900.
00000080:00000010:0.1:1788572250.461076:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342310840.
00000080:00000010:1.1:1788572250.530387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbaa0.
00000080:00000010:1.1:1788572250.530419:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534631d280.
00010000:00000010:1.1:1788572250.540451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572250.540469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:1.1:1788572250.563454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59e0.
00000080:00000010:1.1:1788572250.563483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353308840.
00010000:00000010:1.1:1788572250.569431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572250.569444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:0.1:1788572250.582531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5b20.
00000080:00000010:0.1:1788572250.582563:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352614a40.
00010000:00000010:3.1:1788572250.687490:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572250.687510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572250.704449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4a0.
00000080:00000010:3.1:1788572250.704479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519639c0.
00010000:00000010:3.1:1788572250.720438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572250.720464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572250.731476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d80.
00000080:00000010:3.1:1788572250.731496:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342132940.
00010000:00000010:3.1:1788572250.739911:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572250.739934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572250.749769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86c0.
00000080:00000010:3.1:1788572250.749796:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a7dac0.
00000080:00000010:3.1:1788572250.877694:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb180.
00000080:00000010:3.1:1788572250.877782:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534631e300.
00010000:00000010:3.1:1788572250.889203:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572250.889221:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572250.894495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86c0.
00000080:00000010:3.1:1788572250.894514:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533bd280.
00010000:00000010:3.1:1788572250.901176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572250.901191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572250.916025:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f60.
00000080:00000010:3.1:1788572250.916047:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544de300.
00010000:00000010:3.1:1788572250.929722:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572250.929740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572250.939461:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86c0.
00000080:00000010:3.1:1788572250.939484:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352742100.
00010000:00000010:3.1:1788572250.945538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572250.945551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572250.945555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f60.
00000080:00000010:3.1:1788572250.945571:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519b5ac0.
00010000:00000010:0.1:1788572251.048998:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:0.1:1788572251.049010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:0.1:1788572251.070467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfa0.
00000080:00000010:0.1:1788572251.070495:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535181eb40.
00010000:00000010:0.1:1788572251.079645:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572251.079660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572251.079666:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5800.
00000080:00000010:0.1:1788572251.079682:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354536b40.
00010000:00000010:1.1:1788572251.174090:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:1.1:1788572251.174105:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:0.1:1788572251.186473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb160.
00000080:00000010:0.1:1788572251.186488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bcb9c0.
00010000:00000010:1.1:1788572251.192428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572251.192436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572251.202134:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5820.
00000080:00000010:1.1:1788572251.202171:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346324200.
00010000:00000010:1.1:1788572251.210916:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572251.210936:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572251.219948:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5440.
00000080:00000010:1.1:1788572251.219972:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346017380.
00010000:00000010:1.1:1788572251.228153:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572251.228164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572251.233196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5300.
00000080:00000010:1.1:1788572251.233225:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346336b40.
00010000:00000010:1.1:1788572251.248592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572251.248600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572251.258045:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5440.
00000080:00000010:1.1:1788572251.258064:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421ea100.
00010000:00000010:1.1:1788572251.263449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572251.263464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572251.278594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5300.
00000080:00000010:1.1:1788572251.278618:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601a100.
00010000:00000010:0.1:1788572251.309615:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:0.1:1788572251.309635:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:0.1:1788572251.315513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3a0.
00000080:00000010:0.1:1788572251.315546:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354488840.
00010000:00000010:3.1:1788572251.362525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:3.1:1788572251.362549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572251.388350:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1c0.
00000080:00000010:3.1:1788572251.388374:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535443e300.
00000080:00000010:3.1:1788572251.629659:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:3.1:1788572251.629689:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544b5280.
00010000:00000010:1.1:1788572251.640706:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:1.1:1788572251.640723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:1.1:1788572251.657038:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d00.
00000080:00000010:1.1:1788572251.657063:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a86b40.
00000080:00000010:3.1:1788572251.667969:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00000080:00000010:3.1:1788572251.667992:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353248000.
00010000:00000010:1.1:1788572251.668430:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572251.668444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00010000:00000010:0.1:1788572251.680841:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572251.680849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572251.692527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:0.1:1788572251.692553:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230e300.
00010000:00000010:0.1:1788572251.701456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572251.701463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572251.709916:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55e0.
00000080:00000010:0.1:1788572251.709959:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351916300.
00010000:00000010:0.1:1788572251.714605:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572251.714610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572251.718651:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa50e0.
00000080:00000010:0.1:1788572251.718682:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a58000.
00010000:00000010:0.1:1788572251.728178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572251.728193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572251.748426:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55e0.
00000080:00000010:0.1:1788572251.748556:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494798c0.
00010000:00000010:3.1:1788572251.776954:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572251.776972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572251.779655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb560.
00000080:00000010:3.1:1788572251.779683:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e1080.
00010000:00000010:3.1:1788572251.795531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572251.795546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572251.811721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8760.
00000080:00000010:3.1:1788572251.816628:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b218c0.
00010000:00000010:3.1:1788572251.906432:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572251.906439:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572251.913438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00000080:00000010:3.1:1788572251.913472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a4eb40.
00010000:00000010:3.1:1788572251.917756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572251.917769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:0.1:1788572251.974749:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:0.1:1788572251.974766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572252.013677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:0.1:1788572252.013715:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ac0840.
00010000:00000010:0.1:1788572252.053358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572252.053373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572252.067582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5860.
00000080:00000010:0.1:1788572252.067612:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b7c200.
00010000:00000010:0.1:1788572252.067621:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572252.067627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00010000:00000010:3.1:1788572252.180621:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572252.180641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572252.207310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb000.
00000080:00000010:3.1:1788572252.207343:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519118c0.
00000080:00000010:0.1:1788572252.336797:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe00.
00000080:00000010:0.1:1788572252.336827:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354537380.
00010000:00000010:3.1:1788572252.408652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:3.1:1788572252.408667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572252.422921:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb300.
00000080:00000010:3.1:1788572252.448463:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a5b180.
00000080:00000010:0.1:1788572252.478670:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd60.
00000080:00000010:0.1:1788572252.478826:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346045280.
00010000:00000010:0.1:1788572252.510671:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572252.510875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:0.1:1788572252.529504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba40.
00000080:00000010:0.1:1788572252.529541:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a1b180.
00010000:00000010:0.1:1788572252.549561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572252.549574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572252.570178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5660.
00000080:00000010:0.1:1788572252.570194:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b7a100.
00010000:00000010:3.1:1788572252.598674:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572252.598700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572252.608447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8e0.
00000080:00000010:3.1:1788572252.608476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ee300.
00010000:00000010:3.1:1788572252.608486:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572252.608492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572252.626071:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8660.
00000080:00000010:3.1:1788572252.626103:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519a3180.
00010000:00000010:3.1:1788572252.646000:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572252.646015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572252.646021:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572252.646040:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535322e300.
00010000:00000010:0.1:1788572252.662433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572252.662451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:0.1:1788572252.744664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:0.1:1788572252.744683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:1.1:1788572252.756593:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf80.
00000080:00000010:1.1:1788572252.774447:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346200840.
00010000:00000010:0.1:1788572252.778473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:0.1:1788572252.778487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00010000:00000010:0.1:1788572252.781958:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572252.781970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572252.782407:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5540.
00000080:00000010:1.1:1788572252.782427:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544b0840.
00000080:00000010:0.1:1788572252.792147:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5320.
00000080:00000010:0.1:1788572252.796470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535268b9c0.
00010000:00000010:0.1:1788572252.802946:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572252.802961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572252.811466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:0.1:1788572252.811494:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544cd280.
00010000:00000010:0.1:1788572252.816447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572252.816454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572252.823675:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5480.
00000080:00000010:0.1:1788572252.823975:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535263b180.
00010000:00000010:0.1:1788572252.842635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572252.842642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572252.847972:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:0.1:1788572252.847994:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546fc200.
00010000:00000010:3.1:1788572252.850548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572252.850560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:3.1:1788572252.862415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5480.
00000080:00000010:3.1:1788572252.866858:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aba940.
00010000:00000010:3.1:1788572252.872727:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572252.872744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572252.893271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572252.893299:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230d280.
00010000:00000010:3.1:1788572252.893318:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572252.893324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572252.894735:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8700.
00000080:00000010:3.1:1788572252.894751:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e8840.
00000080:00000010:1.1:1788572252.961430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfc0.
00000080:00000010:1.1:1788572252.961452:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3dac0.
00010000:00000010:1.1:1788572252.979586:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572252.979601:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00010000:00000010:0.1:1788572252.987420:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:0.1:1788572252.987426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572252.987591:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58e0.
00000080:00000010:1.1:1788572252.987605:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353236b40.
00000080:00000010:0.1:1788572252.998450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5420.
00000080:00000010:0.1:1788572252.998471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349546b40.
00010000:00000010:0.1:1788572253.003437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572253.003446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572253.003449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ce0.
00000080:00000010:0.1:1788572253.003460:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494b5ac0.
00000080:00000010:3.1:1788572253.105613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3e0.
00000080:00000010:3.1:1788572253.105633:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351997380.
00010000:00000010:3.1:1788572253.110427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572253.110432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00010000:00000010:3.1:1788572253.176421:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572253.176426:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572253.181593:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:3.1:1788572253.181618:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351996300.
00010000:00000010:3.1:1788572253.186401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572253.186412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572253.191518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572253.191544:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526998c0.
00010000:00000010:3.1:1788572253.195423:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572253.195431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572253.204924:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00000080:00000010:3.1:1788572253.204953:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354475280.
00010000:00000010:3.1:1788572253.217524:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572253.217536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572253.226192:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572253.226215:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bbc200.
00010000:00000010:3.1:1788572253.237646:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572253.237655:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:0.1:1788572253.278642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb680.
00000080:00000010:0.1:1788572253.278656:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534633eb40.
00010000:00000010:0.1:1788572253.325841:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:0.1:1788572253.325857:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:0.1:1788572253.337850:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb620.
00000080:00000010:0.1:1788572253.337869:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534974b180.
00010000:00000010:1.1:1788572253.350523:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:1.1:1788572253.350531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:3.1:1788572253.731980:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8c0.
00000080:00000010:3.1:1788572253.732001:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420bb180.
00010000:00000010:3.1:1788572253.737336:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572253.737353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572253.737358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572253.743733:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ad39c0.
00010000:00000010:3.1:1788572253.751598:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572253.751614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572253.760119:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d89a0.
00000080:00000010:3.1:1788572253.760141:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a5a100.
00010000:00000010:3.1:1788572253.763766:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572253.763783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572253.779432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572253.779448:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953547318c0.
00010000:00000010:3.1:1788572253.888545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572253.888561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572253.900101:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572253.900130:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534217e300.
00010000:00000010:3.1:1788572253.911011:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572253.911027:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572253.922472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:3.1:1788572253.922502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d2100.
00010000:00000010:3.1:1788572254.066583:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:3.1:1788572254.066597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572254.081732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb640.
00000080:00000010:3.1:1788572254.081751:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533e3180.
00010000:00000010:3.1:1788572254.088227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572254.088241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572254.097498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:3.1:1788572254.097516:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a0840.
00010000:00000010:3.1:1788572254.114469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572254.114484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00010000:00000010:1.1:1788572254.188619:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572254.188626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:1.1:1788572254.193447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd00.
00000080:00000010:1.1:1788572254.193472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544bb9c0.
00010000:00000010:1.1:1788572254.210445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572254.210460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572254.225495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53e0.
00000080:00000010:1.1:1788572254.235813:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497a4a40.
00010000:00000010:1.1:1788572254.250245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572254.250259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572254.250264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5120.
00000080:00000010:1.1:1788572254.250279:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351905ac0.
00010000:00000010:1.1:1788572254.257221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.257228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00010000:00000010:3.1:1788572254.390704:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572254.390721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572254.404796:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb420.
00000080:00000010:3.1:1788572254.404813:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947a100.
00010000:00000010:3.1:1788572254.417690:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572254.417708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572254.417714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:3.1:1788572254.417731:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a26300.
00010000:00000010:0.1:1788572254.425874:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:0.1:1788572254.425890:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:0.1:1788572254.436340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8340.
00000080:00000010:0.1:1788572254.439011:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620d280.
00010000:00000010:0.1:1788572254.443429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572254.443453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572254.460633:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5980.
00000080:00000010:0.1:1788572254.460658:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422fe300.
00000080:00000010:3.1:1788572254.487474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb740.
00000080:00000010:3.1:1788572254.487502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351913180.
00010000:00000010:3.1:1788572254.495509:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572254.495525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572254.505924:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:3.1:1788572254.505945:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ac7380.
00010000:00000010:3.1:1788572254.521758:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572254.521764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572254.531127:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8720.
00000080:00000010:3.1:1788572254.531154:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497a5ac0.
00010000:00000010:1.1:1788572254.535639:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.535646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:1.1:1788572254.545053:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:1.1:1788572254.545081:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b7dac0.
00010000:00000010:1.1:1788572254.550778:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.550792:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572254.555874:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59c0.
00000080:00000010:1.1:1788572254.555912:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e5280.
00010000:00000010:1.1:1788572254.563974:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.563991:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572254.575804:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00000080:00000010:1.1:1788572254.575827:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b2dac0.
00010000:00000010:1.1:1788572254.579924:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.579941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572254.591524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59c0.
00000080:00000010:1.1:1788572254.591555:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421beb40.
00010000:00000010:1.1:1788572254.597932:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.597950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572254.604942:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00000080:00000010:1.1:1788572254.608683:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b20840.
00010000:00000010:1.1:1788572254.621125:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.621135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572254.625636:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59c0.
00000080:00000010:1.1:1788572254.635660:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494ea100.
00010000:00000010:1.1:1788572254.644211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.644219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572254.647703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00000080:00000010:1.1:1788572254.647725:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e2940.
00010000:00000010:1.1:1788572254.653375:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.653385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572254.662845:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59c0.
00000080:00000010:1.1:1788572254.662863:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a498c0.
00010000:00000010:1.1:1788572254.666787:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572254.666802:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572254.676196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00000080:00000010:1.1:1788572254.676215:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ab9c0.
00010000:00000010:0.1:1788572254.689890:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:0.1:1788572254.689914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:3.1:1788572254.718819:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5e0.
00000080:00000010:3.1:1788572254.718850:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353248840.
00010000:00000010:3.1:1788572254.722847:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572254.722864:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572254.731494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8720.
00000080:00000010:3.1:1788572254.731582:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230b9c0.
00010000:00000010:1.1:1788572254.768714:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:1.1:1788572254.768730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572254.768884:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0a0.
00000080:00000010:3.1:1788572254.768914:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352703180.
00010000:00000010:0.1:1788572254.838695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:0.1:1788572254.838703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572254.911602:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb60.
00000080:00000010:0.1:1788572254.911620:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a39c0.
00010000:00000010:3.1:1788572254.998876:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572254.998899:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572255.004542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb140.
00000080:00000010:3.1:1788572255.004567:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354736b40.
00010000:00000010:0.1:1788572255.016335:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:0.1:1788572255.016349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:0.1:1788572255.088591:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.088599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572255.093836:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb20.
00000080:00000010:0.1:1788572255.093853:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353379080.
00010000:00000010:0.1:1788572255.104029:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.104045:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572255.109441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa58c0.
00000080:00000010:1.1:1788572255.109470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346313180.
00010000:00000010:1.1:1788572255.163558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:1.1:1788572255.163566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:0.1:1788572255.186687:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:0.1:1788572255.186700:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:0.1:1788572255.191546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb480.
00000080:00000010:0.1:1788572255.191575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353264200.
00010000:00000010:0.1:1788572255.207715:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.207729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572255.217920:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5180.
00000080:00000010:0.1:1788572255.217952:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a4a40.
00000080:00000010:3.1:1788572255.318658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb080.
00000080:00000010:3.1:1788572255.318694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535269ca40.
00010000:00000010:3.1:1788572255.321790:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572255.321799:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572255.329906:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c00.
00000080:00000010:3.1:1788572255.329931:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544739c0.
00010000:00000010:0.1:1788572255.447214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:0.1:1788572255.447228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:0.1:1788572255.455919:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc00.
00000080:00000010:0.1:1788572255.455941:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534974f380.
00010000:00000010:0.1:1788572255.464042:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.464054:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572255.481931:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5de0.
00000080:00000010:0.1:1788572255.481952:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533de300.
00010000:00000010:0.1:1788572255.501459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.501473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572255.501478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53c0.
00000080:00000010:0.1:1788572255.501500:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463398c0.
00010000:00000010:0.1:1788572255.506469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572255.506477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572255.510724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ae0.
00000080:00000010:1.1:1788572255.510748:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421a98c0.
00010000:00000010:0.1:1788572255.518780:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.518793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:0.1:1788572255.523835:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5880.
00000080:00000010:0.1:1788572255.523853:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349746300.
00010000:00000010:0.1:1788572255.533845:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.533853:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572255.542439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53c0.
00000080:00000010:1.1:1788572255.548177:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535447a940.
00010000:00000010:0.1:1788572255.663527:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.663540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:1.1:1788572255.667070:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc80.
00000080:00000010:1.1:1788572255.667097:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354468000.
00010000:00000010:1.1:1788572255.692522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572255.692533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:0.1:1788572255.772070:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb940.
00000080:00000010:0.1:1788572255.772092:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421e5280.
00010000:00000010:0.1:1788572255.835685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.835695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572255.839712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb800.
00000080:00000010:0.1:1788572255.839743:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349749080.
00010000:00000010:0.1:1788572255.860769:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572255.860785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:1.1:1788572255.882462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53e0.
00000080:00000010:1.1:1788572255.882482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b539c0.
00010000:00000010:0.1:1788572256.035376:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572256.035382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:0.1:1788572256.045438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd40.
00000080:00000010:0.1:1788572256.045466:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353236300.
00000080:00000010:0.1:1788572256.178440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd80.
00000080:00000010:0.1:1788572256.178731:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b9e300.
00010000:00000010:3.1:1788572256.307074:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572256.307092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572256.323635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbda0.
00000080:00000010:3.1:1788572256.323653:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422f98c0.
00010000:00000010:3.1:1788572256.329475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572256.329500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:0.1:1788572256.343738:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:0.1:1788572256.343748:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:0.1:1788572256.352692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9a0.
00000080:00000010:0.1:1788572256.352708:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535451d280.
00010000:00000010:0.1:1788572256.361784:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572256.361792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572256.366588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5480.
00000080:00000010:0.1:1788572256.366612:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349564a40.
00010000:00000010:0.1:1788572256.377774:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572256.377788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572256.396924:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:0.1:1788572256.396958:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953547498c0.
00010000:00000010:0.1:1788572256.402630:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572256.402643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572256.410534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5480.
00000080:00000010:0.1:1788572256.410560:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b0a100.
00010000:00000010:1.1:1788572256.419844:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:1.1:1788572256.419859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572256.428448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:0.1:1788572256.428479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354558840.
00010000:00000010:1.1:1788572256.450786:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572256.450794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:1.1:1788572256.459035:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5960.
00000080:00000010:1.1:1788572256.459059:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ac5280.
00010000:00000010:0.1:1788572256.591972:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:0.1:1788572256.591988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572256.666537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb040.
00000080:00000010:3.1:1788572256.666564:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463c0000.
00010000:00000010:3.1:1788572256.831434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572256.831450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572256.854487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe80.
00000080:00000010:3.1:1788572256.854569:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342149080.
00010000:00000010:3.1:1788572256.872052:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572256.872069:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572256.888433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00000080:00000010:3.1:1788572256.888455:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534234a940.
00000080:00000010:3.1:1788572257.013428:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4e0.
00000080:00000010:3.1:1788572257.013452:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546f9080.
00010000:00000010:3.1:1788572257.019340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572257.019353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572257.029027:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00000080:00000010:3.1:1788572257.032856:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534212c200.
00010000:00000010:3.1:1788572257.038438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572257.038455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572257.055130:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8660.
00000080:00000010:3.1:1788572257.055156:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342109080.
00000080:00000010:3.1:1788572257.144500:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfe0.
00000080:00000010:3.1:1788572257.144520:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463da940.
00010000:00000010:3.1:1788572257.149989:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572257.150013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:0.1:1788572257.165490:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8660.
00000080:00000010:0.1:1788572257.170482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534639d280.
00010000:00000010:0.1:1788572257.176117:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572257.176132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572257.188403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a20.
00000080:00000010:0.1:1788572257.192584:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534944d280.
00010000:00000010:1.1:1788572257.213373:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:1.1:1788572257.213387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:3.1:1788572257.285312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1e0.
00000080:00000010:3.1:1788572257.285341:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346395280.
00010000:00000010:3.1:1788572257.307802:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572257.307815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572257.326460:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8400.
00000080:00000010:1.1:1788572257.328429:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532d39c0.
00010000:00000010:1.1:1788572257.334569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572257.334583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572257.352704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5160.
00000080:00000010:1.1:1788572257.354570:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463f8840.
00010000:00000010:1.1:1788572257.373693:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572257.373709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572257.397176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5660.
00000080:00000010:1.1:1788572257.397205:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342348000.
00010000:00000010:1.1:1788572257.404183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572257.404195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572257.417655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5160.
00000080:00000010:1.1:1788572257.417680:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349666300.
00010000:00000010:3.1:1788572257.439299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572257.439313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572257.451046:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5660.
00000080:00000010:3.1:1788572257.451074:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463fd280.
00010000:00000010:3.1:1788572257.469567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572257.469578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00010000:00000010:3.1:1788572257.563617:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572257.563624:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572257.568605:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb80.
00000080:00000010:3.1:1788572257.568631:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546ff380.
00010000:00000010:3.1:1788572257.589074:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572257.589089:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00010000:00000010:1.1:1788572257.681459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572257.681471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:1.1:1788572257.703523:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbcc0.
00000080:00000010:1.1:1788572257.715129:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534600b180.
00010000:00000010:1.1:1788572257.993544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572257.993561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:0.1:1788572258.049436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:0.1:1788572258.049461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:1.1:1788572258.068707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7c0.
00000080:00000010:1.1:1788572258.068734:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535452e300.
00010000:00000010:0.1:1788572258.074303:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572258.074314:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:0.1:1788572258.080414:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a60.
00000080:00000010:0.1:1788572258.080430:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423318c0.
00010000:00000010:0.1:1788572258.089435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572258.089451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572258.097768:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e00.
00000080:00000010:0.1:1788572258.097786:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bc7380.
00010000:00000010:0.1:1788572258.104743:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572258.104757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:1.1:1788572258.115516:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5bc0.
00000080:00000010:1.1:1788572258.115538:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bae300.
00010000:00000010:3.1:1788572258.158063:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572258.158075:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572258.171076:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbca0.
00000080:00000010:3.1:1788572258.173810:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b5ac0.
00010000:00000010:0.1:1788572258.186568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572258.186584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:0.1:1788572258.199656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:0.1:1788572258.199681:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bcf380.
00010000:00000010:0.1:1788572258.209755:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572258.209769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572258.217693:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5980.
00000080:00000010:0.1:1788572258.217720:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534958e300.
00010000:00000010:1.1:1788572258.241949:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:1.1:1788572258.241956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:1.1:1788572258.282351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb760.
00000080:00000010:1.1:1788572258.282369:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544db9c0.
00010000:00000010:1.1:1788572258.296450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572258.296464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572258.304474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5300.
00000080:00000010:1.1:1788572258.304505:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535270a940.
00010000:00000010:1.1:1788572258.313891:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572258.313905:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572258.320422:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5440.
00000080:00000010:1.1:1788572258.320453:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519b9080.
00010000:00000010:3.1:1788572258.352560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:3.1:1788572258.352569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572258.362225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb540.
00000080:00000010:3.1:1788572258.362242:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534631b9c0.
00010000:00000010:3.1:1788572258.385421:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572258.385427:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:0.1:1788572258.443797:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1a0.
00000080:00000010:0.1:1788572258.443825:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ad3180.
00010000:00000010:0.1:1788572258.450569:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572258.450579:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00010000:00000010:0.1:1788572258.549439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:0.1:1788572258.549472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:1.1:1788572258.575894:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb860.
00000080:00000010:1.1:1788572258.575919:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aa4a40.
00010000:00000010:0.1:1788572258.591567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572258.591581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:0.1:1788572258.614314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5560.
00000080:00000010:0.1:1788572258.620599:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a60000.
00010000:00000010:1.1:1788572258.623744:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074f00.
00010000:00000010:1.1:1788572258.623759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:1.1:1788572258.633743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a40.
00000080:00000010:1.1:1788572258.633766:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351af4a40.
00000080:00000010:3.1:1788572258.676753:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5180.
00000080:00000010:3.1:1788572258.676775:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346060000.
00010000:00000010:3.1:1788572258.700884:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572258.700897:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572258.700902:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f60.
00000080:00000010:3.1:1788572258.700918:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463ef380.
00000080:00000010:0.1:1788572258.786089:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2c0.
00000080:00000010:0.1:1788572258.786117:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622b9c0.
00010000:00000010:0.1:1788572258.788651:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572258.788664:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572258.811648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b40.
00000080:00000010:0.1:1788572258.811671:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526118c0.
00010000:00000010:0.1:1788572258.831061:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572258.831068:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572258.831073:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5980.
00000080:00000010:0.1:1788572258.831087:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351af5ac0.
00010000:00000010:0.1:1788572258.842009:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572258.842018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572258.847436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b40.
00000080:00000010:0.1:1788572258.851182:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535337b9c0.
00010000:00000010:1.1:1788572258.871585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:1.1:1788572258.871594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572258.881867:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5980.
00000080:00000010:0.1:1788572258.881884:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953462239c0.
00010000:00000010:1.1:1788572258.897423:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572258.897437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:1.1:1788572258.902439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5800.
00000080:00000010:1.1:1788572258.902481:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460198c0.
00010000:00000010:1.1:1788572258.922687:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572258.922696:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00010000:00000010:3.1:1788572259.010576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572259.010584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:0.1:1788572259.035445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572259.035469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:0.1:1788572259.040756:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc40.
00000080:00000010:0.1:1788572259.040911:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354540000.
00010000:00000010:0.1:1788572259.045615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572259.045628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572259.052446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:0.1:1788572259.052481:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b4a40.
00010000:00000010:0.1:1788572259.065252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572259.065269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572259.093804:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5820.
00000080:00000010:0.1:1788572259.093830:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf4200.
00010000:00000010:0.1:1788572259.093836:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572259.093841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572259.111632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:0.1:1788572259.111656:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535269eb40.
00000080:00000010:3.1:1788572259.244625:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7e0.
00000080:00000010:3.1:1788572259.244785:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352745ac0.
00010000:00000010:3.1:1788572259.261757:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572259.261763:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:0.1:1788572259.266601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d80.
00000080:00000010:0.1:1788572259.266625:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351981080.
00010000:00000010:0.1:1788572259.275440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572259.275467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572259.285454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f20.
00000080:00000010:0.1:1788572259.285485:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b85280.
00010000:00000010:0.1:1788572259.301938:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572259.301951:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572259.334764:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5020.
00000080:00000010:0.1:1788572259.334777:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533bb9c0.
00010000:00000010:3.1:1788572259.378144:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572259.378159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572259.401691:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbde0.
00000080:00000010:3.1:1788572259.401709:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535324a100.
00010000:00000010:3.1:1788572259.498735:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572259.498750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572259.525448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbba0.
00000080:00000010:3.1:1788572259.525476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519a2100.
00010000:00000010:0.1:1788572259.529438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:0.1:1788572259.529455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:0.1:1788572259.555187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87e0.
00000080:00000010:0.1:1788572259.555216:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533e2940.
00010000:00000010:1.1:1788572259.559571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.559581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572259.564452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c20.
00000080:00000010:1.1:1788572259.564573:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352515280.
00010000:00000010:1.1:1788572259.582531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.582540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572259.582545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c60.
00000080:00000010:1.1:1788572259.582559:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527039c0.
00010000:00000010:1.1:1788572259.598730:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.598743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572259.607726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c52a0.
00000080:00000010:1.1:1788572259.607751:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349442940.
00010000:00000010:1.1:1788572259.615557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.615566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572259.620426:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c60.
00000080:00000010:1.1:1788572259.620455:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463c18c0.
00010000:00000010:1.1:1788572259.628461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.628476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572259.636758:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c52a0.
00000080:00000010:1.1:1788572259.636786:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d7380.
00010000:00000010:1.1:1788572259.651180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.651186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572259.656470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c60.
00000080:00000010:1.1:1788572259.656502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be2100.
00010000:00000010:1.1:1788572259.662765:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.662779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572259.672445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c52a0.
00000080:00000010:1.1:1788572259.672479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422dd280.
00010000:00000010:1.1:1788572259.681818:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.681826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572259.686808:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c60.
00000080:00000010:1.1:1788572259.686834:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352513180.
00010000:00000010:1.1:1788572259.703105:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.703121:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572259.709796:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c52a0.
00000080:00000010:1.1:1788572259.709823:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534604a940.
00010000:00000010:1.1:1788572259.731249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572259.731258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572259.737003:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c60.
00000080:00000010:1.1:1788572259.737022:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351acf380.
00010000:00000010:0.1:1788572259.752677:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:0.1:1788572259.752685:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00010000:00000010:0.1:1788572259.845709:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:0.1:1788572259.845725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572259.918641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0e0.
00000080:00000010:3.1:1788572259.918667:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b1a940.
00010000:00000010:3.1:1788572259.930436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572259.930461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572259.944001:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8da0.
00000080:00000010:3.1:1788572259.944028:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535258a100.
00010000:00000010:3.1:1788572259.961006:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572259.961021:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572259.977866:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83e0.
00000080:00000010:3.1:1788572259.977895:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e39c0.
00010000:00000010:3.1:1788572259.986865:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572259.986880:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572260.003067:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8da0.
00010000:00000010:3.1:1788572260.064693:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572260.064700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572260.077776:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb40.
00000080:00000010:3.1:1788572260.077806:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519239c0.
00010000:00000010:1.1:1788572260.266994:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:1.1:1788572260.267001:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00010000:00000010:3.1:1788572260.441861:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572260.441882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572260.465038:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe20.
00000080:00000010:3.1:1788572260.465065:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ab8000.
00010000:00000010:3.1:1788572260.488469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572260.488478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572260.495927:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83e0.
00000080:00000010:3.1:1788572260.495957:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535324eb40.
00010000:00000010:3.1:1788572260.587205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572260.587220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572260.604576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb240.
00000080:00000010:3.1:1788572260.604602:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f98c0.
00010000:00000010:3.1:1788572260.688535:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572260.688552:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572260.708680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7a0.
00000080:00000010:3.1:1788572260.708709:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535443f380.
00000080:00000010:3.1:1788572260.768402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb100.
00000080:00000010:3.1:1788572260.772655:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544e8840.
00010000:00000010:3.1:1788572260.783111:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572260.783122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572260.798435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d60.
00000080:00000010:3.1:1788572260.798462:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349663180.
00010000:00000010:0.1:1788572260.883978:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572260.883990:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:0.1:1788572260.894632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb360.
00000080:00000010:0.1:1788572260.894656:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534214a100.
00010000:00000010:0.1:1788572260.918847:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572260.918860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:1.1:1788572260.919713:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa59a0.
00000080:00000010:1.1:1788572260.919735:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bd1080.
00010000:00000010:1.1:1788572260.929347:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572260.929362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:1.1:1788572260.964701:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5aa0.
00000080:00000010:1.1:1788572260.964726:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349440000.
00010000:00000010:1.1:1788572260.994465:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572260.994479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:1.1:1788572261.008796:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c56e0.
00000080:00000010:1.1:1788572261.008814:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354766b40.
00010000:00000010:1.1:1788572261.017206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572261.017222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:1.1:1788572261.029465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5aa0.
00000080:00000010:1.1:1788572261.029491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349545280.
00010000:00000010:1.1:1788572261.039423:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572261.039429:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:1.1:1788572261.051896:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c56e0.
00000080:00000010:1.1:1788572261.051913:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534602e300.
00010000:00000010:1.1:1788572261.055449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572261.055466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00010000:00000010:3.1:1788572261.078606:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572261.078621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572261.087550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb340.
00000080:00000010:3.1:1788572261.087573:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420bb9c0.
00000080:00000010:3.1:1788572261.281268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5a0.
00000080:00000010:3.1:1788572261.281295:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353295280.
00010000:00000010:0.1:1788572261.290608:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572261.290630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:0.1:1788572261.300628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:0.1:1788572261.305815:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421d1080.
00010000:00000010:0.1:1788572261.313535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572261.313542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572261.322677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5560.
00000080:00000010:0.1:1788572261.322712:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d1080.
00010000:00000010:0.1:1788572261.341451:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572261.341464:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00010000:00000010:3.1:1788572261.489685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572261.489711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:0.1:1788572261.511269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:0.1:1788572261.511284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:3.1:1788572261.609191:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:3.1:1788572261.609205:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572261.618575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb00.
00000080:00000010:3.1:1788572261.618594:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349754a40.
00000080:00000010:0.1:1788572261.690207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbc0.
00000080:00000010:0.1:1788572261.690232:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535275b9c0.
00010000:00000010:0.1:1788572261.696574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572261.696590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572261.697628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ea0.
00000080:00000010:0.1:1788572261.697649:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d7380.
00010000:00000010:0.1:1788572261.708111:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572261.708127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00010000:00000010:3.1:1788572261.742971:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572261.742987:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572261.746813:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2e0.
00000080:00000010:3.1:1788572261.746845:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535475c200.
00010000:00000010:3.1:1788572261.762570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572261.762586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572261.770555:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:3.1:1788572261.770575:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535326e300.
00010000:00000010:0.1:1788572261.774739:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:0.1:1788572261.774755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:0.1:1788572261.791475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8ce0.
00000080:00000010:0.1:1788572261.791502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a2100.
00010000:00000010:0.1:1788572261.848007:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572261.848022:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572261.863445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb280.
00000080:00000010:0.1:1788572261.863469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535441c200.
00010000:00000010:0.1:1788572261.863477:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572261.863481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:0.1:1788572261.873941:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5720.
00000080:00000010:0.1:1788572261.873958:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351978840.
00010000:00000010:0.1:1788572261.879909:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572261.879917:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572261.886059:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5500.
00000080:00000010:0.1:1788572261.886085:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346329080.
00000080:00000010:3.1:1788572262.051436:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534215dac0.
00010000:00000010:1.1:1788572262.059534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572262.059549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:1.1:1788572262.074564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:1.1:1788572262.074586:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354731080.
00010000:00000010:1.1:1788572262.094436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572262.094451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00010000:00000010:3.1:1788572262.141595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:3.1:1788572262.141609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572262.162524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf00.
00000080:00000010:3.1:1788572262.162553:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354408000.
00010000:00000010:3.1:1788572262.245837:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572262.245852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572262.259693:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb460.
00000080:00000010:3.1:1788572262.259710:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b98000.
00010000:00000010:3.1:1788572262.266404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572262.266420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572262.284622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d88c0.
00000080:00000010:3.1:1788572262.284636:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965e300.
00010000:00000010:3.1:1788572262.288456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572262.288467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:3.1:1788572262.299489:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572262.299505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572262.306723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf00.
00000080:00000010:3.1:1788572262.306752:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354496b40.
00010000:00000010:3.1:1788572262.314461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572262.314478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:0.1:1788572262.326551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d88c0.
00000080:00000010:0.1:1788572262.326589:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a47380.
00010000:00000010:1.1:1788572262.330509:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572262.330517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:1.1:1788572262.336266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5d60.
00000080:00000010:1.1:1788572262.336289:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544a0840.
00010000:00000010:1.1:1788572262.340665:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572262.340682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572262.348496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5460.
00000080:00000010:1.1:1788572262.348513:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534600e300.
00010000:00000010:1.1:1788572262.353442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572262.353460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572262.360920:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5560.
00000080:00000010:1.1:1788572262.368536:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a9dac0.
00010000:00000010:0.1:1788572262.372729:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:0.1:1788572262.372746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00010000:00000010:0.1:1788572262.473430:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572262.473453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:0.1:1788572262.473459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb280.
00000080:00000010:0.1:1788572262.473474:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349625280.
00000080:00000010:3.1:1788572262.502458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa54c0.
00000080:00000010:3.1:1788572262.502486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354611080.
00010000:00000010:3.1:1788572262.502494:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572262.502499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:0.1:1788572262.557594:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb00.
00000080:00000010:0.1:1788572262.557628:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463ddac0.
00010000:00000010:0.1:1788572262.565976:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572262.565991:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572262.572696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55c0.
00000080:00000010:0.1:1788572262.572721:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534639b180.
00010000:00000010:0.1:1788572262.576461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572262.576477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572262.599578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c00.
00000080:00000010:0.1:1788572262.599613:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349445ac0.
00010000:00000010:0.1:1788572262.599621:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572262.599625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572262.615568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55c0.
00000080:00000010:0.1:1788572262.615585:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527a18c0.
00010000:00000010:1.1:1788572262.620435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:1.1:1788572262.620448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:3.1:1788572262.647605:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbc0.
00000080:00000010:3.1:1788572262.647628:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532d0840.
00010000:00000010:3.1:1788572262.651054:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572262.651064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572262.654922:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f00.
00000080:00000010:3.1:1788572262.654941:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421a2940.
00010000:00000010:3.1:1788572262.665830:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572262.665843:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:0.1:1788572262.745683:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:0.1:1788572262.745700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:3.1:1788572262.767788:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572262.767804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572262.782476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2e0.
00000080:00000010:3.1:1788572262.782503:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342349080.
00010000:00000010:3.1:1788572262.786684:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572262.786699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572262.798635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f00.
00000080:00000010:3.1:1788572262.798662:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346386b40.
00000080:00000010:0.1:1788572262.836437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5a0.
00000080:00000010:0.1:1788572262.836472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544c8000.
00010000:00000010:3.1:1788572262.848578:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572262.848595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:3.1:1788572262.855770:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5aa0.
00000080:00000010:3.1:1788572262.855794:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519d8840.
00010000:00000010:3.1:1788572262.902714:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572262.902725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572262.922779:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb360.
00000080:00000010:3.1:1788572262.922806:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a8d280.
00010000:00000010:3.1:1788572262.934587:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572262.934594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:0.1:1788572263.038416:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:0.1:1788572263.038551:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:0.1:1788572263.061836:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb340.
00000080:00000010:0.1:1788572263.061863:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965a100.
00010000:00000010:0.1:1788572263.069461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.069477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00010000:00000010:3.1:1788572263.106619:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:3.1:1788572263.106627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:0.1:1788572263.111837:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.111848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572263.113178:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7a0.
00000080:00000010:3.1:1788572263.113197:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349561080.
00010000:00000010:1.1:1788572263.179537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:1.1:1788572263.179551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:0.1:1788572263.185875:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb100.
00000080:00000010:0.1:1788572263.185904:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a44200.
00010000:00000010:0.1:1788572263.191633:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.191641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572263.197873:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5420.
00000080:00000010:0.1:1788572263.197892:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a139c0.
00010000:00000010:0.1:1788572263.216775:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.216792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00010000:00000010:3.1:1788572263.230633:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572263.230642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572263.236244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb240.
00000080:00000010:3.1:1788572263.236273:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534621b180.
00010000:00000010:3.1:1788572263.252181:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572263.252191:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:0.1:1788572263.318803:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.318813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:1.1:1788572263.323880:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb40.
00000080:00000010:1.1:1788572263.323910:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346016300.
00010000:00000010:0.1:1788572263.345630:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.345648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00010000:00000010:0.1:1788572263.445520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.445527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:3.1:1788572263.494559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572263.494574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572263.500978:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0e0.
00000080:00000010:3.1:1788572263.505054:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351984a40.
00010000:00000010:3.1:1788572263.508716:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572263.508732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572263.527063:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d81e0.
00000080:00000010:3.1:1788572263.527083:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534217b180.
00010000:00000010:3.1:1788572263.553514:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572263.553529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572263.587067:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8640.
00000080:00000010:3.1:1788572263.587095:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b4200.
00000080:00000010:0.1:1788572263.632811:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe20.
00000080:00000010:0.1:1788572263.632847:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b6eb40.
00010000:00000010:0.1:1788572263.638277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:0.1:1788572263.638285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00010000:00000010:0.1:1788572263.709298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.709306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00000080:00000010:0.1:1788572263.725538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbde0.
00000080:00000010:0.1:1788572263.725561:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346060840.
00010000:00000010:0.1:1788572263.735314:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.735328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:3.1:1788572263.744468:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e2100.
00010000:00000010:3.1:1788572263.762650:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572263.762667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:1.1:1788572263.778434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8640.
00000080:00000010:1.1:1788572263.778451:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463af380.
00000080:00000010:0.1:1788572263.841436:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbba0.
00000080:00000010:0.1:1788572263.841457:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346037380.
00010000:00000010:1.1:1788572263.857874:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572263.857886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00010000:00000010:0.1:1788572263.864890:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572263.864900:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572263.867827:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5660.
00000080:00000010:1.1:1788572263.867846:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aa0840.
00000080:00000010:0.1:1788572263.873073:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58a0.
00000080:00000010:0.1:1788572263.873103:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346030000.
00010000:00000010:1.1:1788572263.880948:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572263.880958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:1.1:1788572263.884945:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5640.
00000080:00000010:1.1:1788572263.884968:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353309080.
00010000:00000010:1.1:1788572263.896575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572263.896597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572263.907435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53a0.
00000080:00000010:1.1:1788572263.907464:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535256d280.
00010000:00000010:1.1:1788572263.926550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572263.926557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:3.1:1788572264.008950:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b918c0.
00010000:00000010:3.1:1788572264.018923:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572264.018931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572264.018936:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f40.
00000080:00000010:3.1:1788572264.018950:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526a9080.
00010000:00000010:1.1:1788572264.038896:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:1.1:1788572264.038918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:1.1:1788572264.062806:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2c0.
00000080:00000010:1.1:1788572264.069597:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a68000.
00000080:00000010:3.1:1788572264.091058:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f40.
00000080:00000010:3.1:1788572264.091083:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495feb40.
00010000:00000010:0.1:1788572264.162952:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:0.1:1788572264.162967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:0.1:1788572264.172820:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc40.
00000080:00000010:0.1:1788572264.172845:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342130000.
00010000:00000010:0.1:1788572264.184785:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572264.184794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:1.1:1788572264.294381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7e0.
00000080:00000010:1.1:1788572264.294736:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421e8840.
00010000:00000010:1.1:1788572264.298769:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572264.298784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572264.309984:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c52e0.
00000080:00000010:1.1:1788572264.310009:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354481080.
00010000:00000010:1.1:1788572264.316851:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572264.316865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572264.335801:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a80.
00000080:00000010:1.1:1788572264.335864:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601a940.
00010000:00000010:1.1:1788572264.342020:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572264.342035:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572264.353844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c52e0.
00000080:00000010:1.1:1788572264.353864:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519d98c0.
00010000:00000010:3.1:1788572264.361517:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572264.361529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:3.1:1788572264.376912:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a80.
00000080:00000010:3.1:1788572264.376929:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544deb40.
00000080:00000010:0.1:1788572264.423505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb860.
00000080:00000010:0.1:1788572264.423522:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519be300.
00010000:00000010:0.1:1788572264.429573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572264.429590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572264.438333:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:0.1:1788572264.438363:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535193eb40.
00010000:00000010:1.1:1788572264.451555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:1.1:1788572264.451574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00010000:00000010:3.1:1788572264.497993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572264.498011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572264.501570:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb540.
00000080:00000010:3.1:1788572264.501587:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349761080.
00010000:00000010:3.1:1788572264.508046:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572264.508062:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572264.524199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00010000:00000010:3.1:1788572264.567576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:3.1:1788572264.567601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572264.581315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1a0.
00000080:00000010:3.1:1788572264.591781:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ab180.
00010000:00000010:3.1:1788572264.596019:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572264.596029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572264.605975:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00000080:00000010:3.1:1788572264.616678:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353265280.
00010000:00000010:0.1:1788572264.627966:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572264.627974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00010000:00000010:0.1:1788572264.639509:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572264.639524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572264.679827:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbca0.
00000080:00000010:3.1:1788572264.679854:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b418c0.
00010000:00000010:3.1:1788572264.738052:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572264.738068:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572264.753719:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7c0.
00000080:00000010:3.1:1788572264.753743:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346010840.
00010000:00000010:3.1:1788572264.753750:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572264.753755:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572264.759736:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8a80.
00000080:00000010:3.1:1788572264.759762:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497ae300.
00010000:00000010:3.1:1788572264.771065:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572264.771073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572264.777593:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:3.1:1788572264.777621:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346215ac0.
00010000:00000010:0.1:1788572264.848566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:0.1:1788572264.848573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572264.918577:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb760.
00000080:00000010:3.1:1788572264.918653:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519d9080.
00010000:00000010:3.1:1788572264.928931:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572264.928940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572264.935630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:3.1:1788572264.935680:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b3180.
00000080:00000010:0.1:1788572265.010791:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb80.
00000080:00000010:0.1:1788572265.010822:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351980000.
00010000:00000010:0.1:1788572265.024440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572265.024464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572265.031844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53c0.
00000080:00000010:0.1:1788572265.031870:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351999080.
00010000:00000010:0.1:1788572265.042438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572265.042453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:1.1:1788572265.053447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa59e0.
00000080:00000010:1.1:1788572265.053472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535193d280.
00010000:00000010:1.1:1788572265.053481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572265.053486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572265.059233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5da0.
00000080:00000010:1.1:1788572265.059257:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534967c200.
00010000:00000010:1.1:1788572265.062569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572265.062586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572265.085132:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5440.
00000080:00000010:1.1:1788572265.085157:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b92940.
00010000:00000010:1.1:1788572265.107602:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572265.107618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00010000:00000010:3.1:1788572265.281061:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572265.281074:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572265.281080:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbcc0.
00000080:00000010:3.1:1788572265.281097:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460318c0.
00010000:00000010:3.1:1788572265.289531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572265.289546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572265.324754:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:3.1:1788572265.324781:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526aeb40.
00000080:00000010:0.1:1788572265.366777:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1e0.
00000080:00000010:0.1:1788572265.374893:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b91080.
00010000:00000010:0.1:1788572265.381551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572265.381568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572265.384821:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56c0.
00000080:00000010:0.1:1788572265.384840:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e7380.
00010000:00000010:0.1:1788572265.389760:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572265.389775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572265.415734:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5420.
00000080:00000010:0.1:1788572265.415761:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342136300.
00010000:00000010:1.1:1788572265.420648:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:1.1:1788572265.420670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00010000:00000010:3.1:1788572265.444218:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:3.1:1788572265.444232:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572265.465188:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfe0.
00000080:00000010:3.1:1788572265.465209:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546cf380.
00010000:00000010:3.1:1788572265.469863:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572265.469870:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00010000:00000010:0.1:1788572265.533715:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:0.1:1788572265.533733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:0.1:1788572265.553462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe80.
00000080:00000010:0.1:1788572265.553506:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a6ca40.
00010000:00000010:0.1:1788572265.579737:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572265.584526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572265.602667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa58c0.
00000080:00000010:0.1:1788572265.602693:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495f8840.
00010000:00000010:0.1:1788572265.634447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572265.634462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572265.649928:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5220.
00000080:00000010:0.1:1788572265.649949:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352614200.
00010000:00000010:0.1:1788572265.673558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572265.673582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572265.683699:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa58c0.
00000080:00000010:0.1:1788572265.688451:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533098c0.
00010000:00000010:0.1:1788572265.712213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572265.712229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:3.1:1788572265.776746:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354487380.
00010000:00000010:3.1:1788572265.791683:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572265.791706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572265.823492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8cc0.
00000080:00000010:3.1:1788572265.823522:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354484200.
00000080:00000010:0.1:1788572265.868436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8280.
00000080:00000010:0.1:1788572265.868462:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421ee300.
00010000:00000010:1.1:1788572265.878693:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572265.878706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:1.1:1788572265.886042:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa58e0.
00000080:00000010:1.1:1788572265.886061:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346335280.
00010000:00000010:1.1:1788572265.893493:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074900.
00010000:00000010:1.1:1788572265.893508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:1.1:1788572265.903057:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5fa0.
00000080:00000010:1.1:1788572265.903074:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463239c0.
00000080:00000010:3.1:1788572265.980725:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb040.
00000080:00000010:3.1:1788572265.980745:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b6d280.
00010000:00000010:3.1:1788572266.000455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572266.000471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572266.000477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8cc0.
00000080:00000010:3.1:1788572266.000492:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bbdac0.
00010000:00000010:3.1:1788572266.176875:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572266.176887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572266.181481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9a0.
00000080:00000010:3.1:1788572266.181507:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518f39c0.
00010000:00000010:0.1:1788572266.287922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:0.1:1788572266.287936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:0.1:1788572266.295110:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4e0.
00000080:00000010:0.1:1788572266.295134:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535181dac0.
00010000:00000010:0.1:1788572266.302872:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572266.302887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572266.311921:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e00.
00000080:00000010:0.1:1788572266.311939:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544da940.
00010000:00000010:0.1:1788572266.318825:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572266.318839:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572266.328166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5bc0.
00000080:00000010:0.1:1788572266.328184:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533beb40.
00010000:00000010:1.1:1788572266.337746:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:1.1:1788572266.337761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00010000:00000010:1.1:1788572266.353601:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572266.358462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:1.1:1788572266.373775:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbda0.
00000080:00000010:1.1:1788572266.373802:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a9a940.
00010000:00000010:1.1:1788572266.384072:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074800.
00010000:00000010:1.1:1788572266.384081:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572266.394529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5940.
00000080:00000010:1.1:1788572266.400453:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b34200.
00010000:00000010:0.1:1788572266.404808:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572266.404825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:0.1:1788572266.418905:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c60.
00000080:00000010:0.1:1788572266.418932:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953545598c0.
00010000:00000010:0.1:1788572266.422735:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572266.422749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572266.430189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5340.
00000080:00000010:0.1:1788572266.430213:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a44a40.
00010000:00000010:0.1:1788572266.445802:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572266.445817:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572266.454434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b40.
00000080:00000010:0.1:1788572266.454458:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544a2100.
00010000:00000010:0.1:1788572266.469520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572266.469535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572266.477981:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5340.
00000080:00000010:0.1:1788572266.478005:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965a940.
00010000:00000010:3.1:1788572266.554809:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572266.554824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572266.568347:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd40.
00000080:00000010:3.1:1788572266.568364:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ae4200.
00010000:00000010:0.1:1788572266.634461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:0.1:1788572266.634477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:1.1:1788572266.646783:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb800.
00000080:00000010:1.1:1788572266.646817:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527a7380.
00010000:00000010:3.1:1788572266.689901:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572266.689928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:0.1:1788572266.759492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:0.1:1788572266.759508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:1.1:1788572266.792538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb940.
00000080:00000010:1.1:1788572266.792576:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d4200.
00000080:00000010:3.1:1788572266.943344:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc80.
00000080:00000010:3.1:1788572266.943375:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349490000.
00010000:00000010:3.1:1788572266.943515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572266.943523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572266.958226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572266.958253:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354614a40.
00000080:00000010:3.1:1788572267.036381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc00.
00000080:00000010:3.1:1788572267.047160:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342120000.
00010000:00000010:0.1:1788572267.092968:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:0.1:1788572267.092981:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00010000:00000010:0.1:1788572267.162890:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:0.1:1788572267.162909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:1.1:1788572267.187547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb080.
00000080:00000010:1.1:1788572267.189856:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346305280.
00010000:00000010:0.1:1788572267.202437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572267.202486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:0.1:1788572267.222357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5000.
00000080:00000010:0.1:1788572267.222414:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535268e300.
00010000:00000010:0.1:1788572267.237487:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572267.237501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:1.1:1788572267.245482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5340.
00000080:00000010:1.1:1788572267.245506:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534639dac0.
00010000:00000010:1.1:1788572267.442441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074f00.
00010000:00000010:1.1:1788572267.442458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:1.1:1788572267.459458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd80.
00000080:00000010:1.1:1788572267.459484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342345ac0.
00010000:00000010:1.1:1788572267.485854:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074f00.
00010000:00000010:1.1:1788572267.485870:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572267.524439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5fa0.
00000080:00000010:1.1:1788572267.528560:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349636b40.
00010000:00000010:1.1:1788572267.551700:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074f00.
00010000:00000010:1.1:1788572267.551717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:0.1:1788572267.627092:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572267.627109:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:1.1:1788572267.636920:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb20.
00000080:00000010:1.1:1788572267.636946:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546f0000.
00010000:00000010:0.1:1788572267.670228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572267.670244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:0.1:1788572267.689372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d20.
00000080:00000010:0.1:1788572267.689423:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346208840.
00010000:00000010:0.1:1788572267.710967:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572267.710974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572267.717081:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5160.
00000080:00000010:0.1:1788572267.717106:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b32100.
00010000:00000010:0.1:1788572267.762431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572267.762446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572267.769627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55a0.
00000080:00000010:0.1:1788572267.769653:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534600a100.
00010000:00000010:0.1:1788572267.790515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572267.790540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:1.1:1788572267.820437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5160.
00000080:00000010:1.1:1788572267.820475:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b0eb40.
00010000:00000010:0.1:1788572267.977985:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:0.1:1788572267.978002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:1.1:1788572267.993879:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb480.
00000080:00000010:1.1:1788572267.993898:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965b9c0.
00010000:00000010:0.1:1788572268.014654:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572268.014669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:0.1:1788572268.014674:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5860.
00000080:00000010:0.1:1788572268.014688:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463b8840.
00010000:00000010:3.1:1788572268.014732:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572268.014737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:3.1:1788572268.037936:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ba0.
00000080:00000010:3.1:1788572268.037958:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535453ca40.
00010000:00000010:3.1:1788572268.048734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572268.048750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572268.056699:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572268.061451:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535190eb40.
00010000:00000010:3.1:1788572268.067773:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572268.067796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572268.108503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8540.
00000080:00000010:3.1:1788572268.108528:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a14200.
00010000:00000010:3.1:1788572268.130914:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572268.130922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572268.140045:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572268.140073:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a698c0.
00010000:00000010:0.1:1788572268.221722:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:0.1:1788572268.221736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:0.1:1788572268.225512:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb140.
00010000:00000010:3.1:1788572268.398481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:3.1:1788572268.398497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572268.407834:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0a0.
00000080:00000010:3.1:1788572268.407856:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b73180.
00010000:00000010:3.1:1788572268.425568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572268.425582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572268.459045:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572268.459075:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353296300.
00010000:00000010:0.1:1788572268.468466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:0.1:1788572268.468484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00000080:00000010:0.1:1788572268.476427:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00000080:00000010:0.1:1788572268.476443:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bfa940.
00010000:00000010:0.1:1788572268.485637:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572268.485649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572268.492477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa51e0.
00000080:00000010:0.1:1788572268.492500:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535197e300.
00000080:00000010:3.1:1788572268.551545:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb6300.
00010000:00000010:3.1:1788572268.568520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572268.568528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572268.574743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572268.574761:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342087380.
00010000:00000010:3.1:1788572268.608345:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572268.608353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572268.616624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fc0.
00000080:00000010:3.1:1788572268.616649:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351970000.
00010000:00000010:3.1:1788572268.620740:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572268.620756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572268.659275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572268.659305:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535199ca40.
00010000:00000010:0.1:1788572268.670485:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:0.1:1788572268.670494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:0.1:1788572268.676544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fc0.
00000080:00000010:0.1:1788572268.676587:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535198b180.
00010000:00000010:0.1:1788572268.694481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572268.694497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572268.709536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c60.
00000080:00000010:0.1:1788572268.709558:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544698c0.
00010000:00000010:0.1:1788572268.724540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572268.724550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572268.733631:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5040.
00000080:00000010:0.1:1788572268.733650:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d2100.
00010000:00000010:1.1:1788572268.749510:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:1.1:1788572268.749518:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:3.1:1788572268.867427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5e0.
00000080:00000010:3.1:1788572268.867465:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342182100.
00010000:00000010:3.1:1788572268.882659:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572268.882673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00010000:00000010:3.1:1788572268.984800:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:3.1:1788572268.984815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572269.005458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb740.
00000080:00000010:3.1:1788572269.005483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535441ca40.
00010000:00000010:1.1:1788572269.024194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572269.024204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:1.1:1788572269.045831:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8280.
00000080:00000010:1.1:1788572269.045850:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb4a40.
00010000:00000010:1.1:1788572269.049931:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572269.049948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:1.1:1788572269.088643:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54a0.
00000080:00000010:1.1:1788572269.088670:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535447dac0.
00010000:00000010:0.1:1788572269.181746:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572269.181762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:0.1:1788572269.199475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb60.
00000080:00000010:0.1:1788572269.199500:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535275c200.
00010000:00000010:1.1:1788572269.223477:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:1.1:1788572269.223494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00010000:00000010:1.1:1788572269.318181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572269.318196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:1.1:1788572269.334737:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd00.
00000080:00000010:1.1:1788572269.334773:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349741080.
00010000:00000010:0.1:1788572269.350631:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572269.350638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00010000:00000010:0.1:1788572269.455516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:0.1:1788572269.455532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:3.1:1788572269.561922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:3.1:1788572269.561938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572269.584624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb640.
00000080:00000010:3.1:1788572269.588154:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a55280.
00010000:00000010:3.1:1788572269.658605:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572269.658620:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572269.691837:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8c0.
00000080:00000010:3.1:1788572269.691852:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353232100.
00010000:00000010:3.1:1788572269.694463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572269.694478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:1.1:1788572269.851459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:1.1:1788572269.851474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:1.1:1788572269.862447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb620.
00000080:00000010:1.1:1788572269.862472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b739c0.
00010000:00000010:1.1:1788572269.871288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572269.871304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572269.890587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d20.
00000080:00000010:1.1:1788572269.890604:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353244200.
00010000:00000010:1.1:1788572269.895477:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572269.895491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572269.895496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5280.
00000080:00000010:1.1:1788572269.895511:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535324d280.
00010000:00000010:1.1:1788572269.907867:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572269.907873:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572269.923843:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f20.
00000080:00000010:1.1:1788572269.923856:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351abd280.
00010000:00000010:1.1:1788572269.931967:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572269.931975:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572269.940881:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5280.
00000080:00000010:1.1:1788572269.940903:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352510000.
00010000:00000010:1.1:1788572269.949801:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572269.949808:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572269.965988:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d20.
00000080:00000010:1.1:1788572269.966007:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d3180.
00010000:00000010:1.1:1788572269.973455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572269.973468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572269.977459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5280.
00000080:00000010:1.1:1788572269.977476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422ddac0.
00010000:00000010:1.1:1788572269.977482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572269.977486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572269.983433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f20.
00000080:00000010:1.1:1788572269.983464:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349443180.
00010000:00000010:1.1:1788572269.998587:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572269.998600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572270.003155:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d20.
00000080:00000010:1.1:1788572270.003172:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bd2100.
00010000:00000010:1.1:1788572270.012226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572270.012232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572270.016432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f20.
00000080:00000010:1.1:1788572270.016454:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534639a100.
00010000:00000010:1.1:1788572270.028963:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:1.1:1788572270.028971:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572270.035318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d20.
00000080:00000010:1.1:1788572270.035346:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ed280.
00010000:00000010:0.1:1788572270.054889:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074500.
00010000:00000010:0.1:1788572270.054904:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572270.058465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f20.
00000080:00000010:1.1:1788572270.058488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352515ac0.
00010000:00000010:0.1:1788572270.070037:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572270.070053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572270.094722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5280.
00000080:00000010:0.1:1788572270.094750:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519a1080.
00010000:00000010:0.1:1788572270.107640:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572270.107653:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:3.1:1788572270.125612:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5740.
00000080:00000010:3.1:1788572270.125636:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342360840.
00010000:00000010:3.1:1788572270.301942:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572270.301950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572270.318549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb680.
00000080:00000010:3.1:1788572270.318576:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497a98c0.
00010000:00000010:0.1:1788572270.345438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572270.345454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572270.372780:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f60.
00000080:00000010:0.1:1788572270.372807:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e4200.
00010000:00000010:0.1:1788572270.394725:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572270.394741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572270.400871:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5660.
00000080:00000010:0.1:1788572270.400903:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351acca40.
00010000:00000010:0.1:1788572270.439021:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572270.439034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572270.445844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ca0.
00000080:00000010:0.1:1788572270.445870:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497a4200.
00010000:00000010:0.1:1788572270.466148:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572270.466156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572270.470460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5660.
00000080:00000010:0.1:1788572270.470483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e0000.
00010000:00000010:0.1:1788572270.514043:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572270.514054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572270.514060:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ca0.
00000080:00000010:0.1:1788572270.514075:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a48000.
00010000:00000010:0.1:1788572270.518688:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572270.518704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572270.548436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5660.
00000080:00000010:0.1:1788572270.571823:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535258f380.
00010000:00000010:0.1:1788572270.578664:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572270.578678:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572270.582805:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ca0.
00000080:00000010:0.1:1788572270.582830:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b1b180.
00010000:00000010:0.1:1788572270.602662:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572270.602675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572270.638462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5660.
00000080:00000010:0.1:1788572270.638491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b2ca40.
00010000:00000010:0.1:1788572270.659741:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572270.659749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572270.664439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ca0.
00000080:00000010:0.1:1788572270.664518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533c39c0.
00010000:00000010:0.1:1788572270.676637:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572270.676652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572270.683439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5660.
00000080:00000010:0.1:1788572270.683471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534602a940.
00010000:00000010:0.1:1788572270.700652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572270.700659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572270.970903:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3e0.
00000080:00000010:0.1:1788572270.975553:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494ea940.
00010000:00000010:3.1:1788572271.075439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572271.075462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572271.090654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb420.
00000080:00000010:3.1:1788572271.090682:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534222a100.
00010000:00000010:3.1:1788572271.141487:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572271.141505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572271.155455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8a80.
00000080:00000010:3.1:1788572271.155483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b11080.
00010000:00000010:0.1:1788572271.283888:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572271.283902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:3.1:1788572271.462969:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572271.462983:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572271.466620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfc0.
00000080:00000010:3.1:1788572271.466640:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544bb180.
00010000:00000010:3.1:1788572271.471702:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572271.471709:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00010000:00000010:3.1:1788572271.601446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572271.601475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572271.608565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf80.
00000080:00000010:3.1:1788572271.608591:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601b9c0.
00010000:00000010:3.1:1788572271.620457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572271.620472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572271.627681:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8880.
00000080:00000010:3.1:1788572271.627704:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535448eb40.
00010000:00000010:3.1:1788572271.753997:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572271.754007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572271.767037:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8e0.
00000080:00000010:3.1:1788572271.767062:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544d5ac0.
00000080:00000010:1.1:1788572271.981492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:1.1:1788572271.981529:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534631c200.
00010000:00000010:1.1:1788572271.985841:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572271.985848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572271.990802:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5860.
00000080:00000010:1.1:1788572271.990819:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351926b40.
00010000:00000010:1.1:1788572271.994517:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572271.994790:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572271.999549:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5660.
00000080:00000010:1.1:1788572271.999564:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421318c0.
00010000:00000010:1.1:1788572272.004654:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572272.004662:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572272.009417:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5860.
00000080:00000010:1.1:1788572272.013429:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a61080.
00010000:00000010:1.1:1788572272.017463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572272.017477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:0.1:1788572272.022440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5660.
00000080:00000010:0.1:1788572272.022473:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535256c200.
00010000:00000010:3.1:1788572272.105753:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572272.105773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572272.114477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd60.
00000080:00000010:3.1:1788572272.114523:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346220840.
00010000:00000010:3.1:1788572272.130943:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572272.130958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:0.1:1788572272.221791:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb300.
00000080:00000010:0.1:1788572272.221818:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351966300.
00010000:00000010:0.1:1788572272.230443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572272.230458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572272.239619:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52e0.
00000080:00000010:0.1:1788572272.239649:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a60840.
00010000:00000010:0.1:1788572272.245456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572272.245466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572272.250701:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ee0.
00000080:00000010:0.1:1788572272.250724:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534218a940.
00010000:00000010:0.1:1788572272.260456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572272.260473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572272.282578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52e0.
00000080:00000010:0.1:1788572272.282605:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533d5280.
00010000:00000010:0.1:1788572272.305467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572272.305486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00010000:00000010:3.1:1788572272.423915:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:3.1:1788572272.423931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572272.430914:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe00.
00000080:00000010:3.1:1788572272.430940:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b46b40.
00010000:00000010:3.1:1788572272.444542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572272.444557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572272.455430:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8140.
00000080:00000010:3.1:1788572272.455447:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518d5280.
00010000:00000010:3.1:1788572272.481758:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572272.481772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:0.1:1788572272.486436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8640.
00000080:00000010:0.1:1788572272.486454:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b8840.
00010000:00000010:1.1:1788572272.492232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:1.1:1788572272.492250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00010000:00000010:0.1:1788572272.531428:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572272.531435:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:1.1:1788572272.557882:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb000.
00000080:00000010:1.1:1788572272.557903:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351985ac0.
00010000:00000010:3.1:1788572272.576757:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572272.576769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00010000:00000010:3.1:1788572272.633531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572272.633540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572272.712498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba40.
00000080:00000010:0.1:1788572272.712538:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354541080.
00010000:00000010:3.1:1788572272.740458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:3.1:1788572272.740476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572272.766191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:3.1:1788572272.766208:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535455b180.
00010000:00000010:3.1:1788572272.766221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572272.766225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572272.787363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8140.
00000080:00000010:3.1:1788572272.787406:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474e300.
00000080:00000010:1.1:1788572272.850469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00000080:00000010:1.1:1788572272.850485:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953545198c0.
00010000:00000010:1.1:1788572272.864061:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572272.864072:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00010000:00000010:0.1:1788572272.874868:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572272.874874:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572272.898432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb560.
00000080:00000010:0.1:1788572272.898660:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965eb40.
00010000:00000010:3.1:1788572272.954818:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572272.954834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572272.961476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:3.1:1788572272.961500:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535190e300.
00010000:00000010:3.1:1788572273.217453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572273.217470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572273.226433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1c0.
00000080:00000010:3.1:1788572273.226449:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535268f380.
00000080:00000010:0.1:1788572273.244440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d80a0.
00000080:00000010:0.1:1788572273.250182:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534212ca40.
00010000:00000010:3.1:1788572273.366820:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:3.1:1788572273.366837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572273.383820:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3a0.
00000080:00000010:3.1:1788572273.391540:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495aa100.
00000080:00000010:0.1:1788572273.489863:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb160.
00000080:00000010:0.1:1788572273.489888:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463feb40.
00010000:00000010:0.1:1788572273.504240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572273.504257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572273.517716:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5700.
00000080:00000010:0.1:1788572273.517745:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532d5ac0.
00010000:00000010:0.1:1788572273.524087:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572273.524103:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572273.543739:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5780.
00000080:00000010:0.1:1788572273.543757:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463918c0.
00000080:00000010:3.1:1788572273.634601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfa0.
00000080:00000010:3.1:1788572273.640215:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535268c200.
00010000:00000010:3.1:1788572273.654636:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572273.654651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572273.659814:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572273.659841:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d18c0.
00010000:00000010:3.1:1788572273.681717:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572273.681745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572273.724187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f00.
00000080:00000010:3.1:1788572273.724208:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba7380.
00010000:00000010:3.1:1788572273.732450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572273.732468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572273.753450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572273.757382:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534210dac0.
00010000:00000010:3.1:1788572273.765854:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572273.765862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572273.777595:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f00.
00000080:00000010:3.1:1788572273.777617:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a82100.
00000080:00000010:0.1:1788572273.937443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4a0.
00000080:00000010:0.1:1788572273.937473:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947b180.
00010000:00000010:0.1:1788572273.947280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572273.947294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00010000:00000010:1.1:1788572274.012500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572274.012516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572274.017465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbaa0.
00000080:00000010:1.1:1788572274.017497:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534600d280.
00010000:00000010:1.1:1788572274.029344:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572274.029359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572274.044862:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5980.
00000080:00000010:1.1:1788572274.044889:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b3e300.
00010000:00000010:0.1:1788572274.057206:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:0.1:1788572274.057223:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:0.1:1788572274.072243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:0.1:1788572274.072257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:0.1:1788572274.075463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb180.
00000080:00000010:0.1:1788572274.075489:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a1f380.
00010000:00000010:0.1:1788572274.087139:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572274.087155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572274.101236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a80.
00000080:00000010:0.1:1788572274.101265:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ac5ac0.
00010000:00000010:0.1:1788572274.116190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572274.116205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:1.1:1788572274.126509:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ae2100.
00000080:00000010:0.1:1788572274.130459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ce0.
00010000:00000010:0.1:1788572274.135444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572274.135459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:0.1:1788572274.148884:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5b80.
00000080:00000010:1.1:1788572274.151450:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534634c200.
00000080:00000010:1.1:1788572274.281477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb960.
00000080:00000010:1.1:1788572274.281517:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420bc200.
00010000:00000010:0.1:1788572274.292731:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:0.1:1788572274.292748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:0.1:1788572274.306674:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d80.
00000080:00000010:0.1:1788572274.306698:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354737380.
00010000:00000010:0.1:1788572274.323695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572274.323713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572274.323718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ce0.
00000080:00000010:0.1:1788572274.323741:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342158840.
00010000:00000010:0.1:1788572274.330669:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572274.330685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00010000:00000010:0.1:1788572274.479113:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572274.479121:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:0.1:1788572274.512430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:0.1:1788572274.512553:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d5280.
00010000:00000010:1.1:1788572274.515763:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572274.515776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:1.1:1788572274.519816:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a60.
00000080:00000010:1.1:1788572274.519834:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354474200.
00010000:00000010:1.1:1788572274.529432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572274.529439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572274.543664:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57e0.
00000080:00000010:1.1:1788572274.543684:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535441e300.
00010000:00000010:1.1:1788572274.548648:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572274.548665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572274.563612:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54e0.
00000080:00000010:1.1:1788572274.563636:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba39c0.
00010000:00000010:1.1:1788572274.573472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572274.573486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572274.581761:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57e0.
00000080:00000010:1.1:1788572274.581786:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544798c0.
00010000:00000010:0.1:1788572274.593631:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:0.1:1788572274.593638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:0.1:1788572274.607714:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54e0.
00000080:00000010:0.1:1788572274.607742:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353266b40.
00010000:00000010:0.1:1788572274.619758:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572274.619771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572274.626452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa59a0.
00000080:00000010:0.1:1788572274.626478:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a4200.
00010000:00000010:0.1:1788572274.635448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572274.635465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572274.659647:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5da0.
00000080:00000010:0.1:1788572274.659670:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf98c0.
00010000:00000010:0.1:1788572274.667195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572274.667211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572274.671448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa59a0.
00000080:00000010:0.1:1788572274.671477:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534958a940.
00000080:00000010:3.1:1788572274.707538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5da0.
00000080:00000010:3.1:1788572274.707562:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b5b180.
00010000:00000010:3.1:1788572274.711764:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572274.711771:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572274.720437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f00.
00000080:00000010:3.1:1788572274.720466:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534218f380.
00010000:00000010:3.1:1788572274.724438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572274.724460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572274.743470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8100.
00000080:00000010:3.1:1788572274.743506:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421ad280.
00010000:00000010:0.1:1788572274.866589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:0.1:1788572274.866605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572274.904448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:3.1:1788572274.904476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351994a40.
00010000:00000010:3.1:1788572274.916735:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572274.916745:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572274.925753:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8100.
00000080:00000010:3.1:1788572274.925783:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535446b180.
00010000:00000010:0.1:1788572274.938610:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:0.1:1788572274.938741:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:0.1:1788572274.946888:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8be0.
00000080:00000010:1.1:1788572274.954770:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495deb40.
00010000:00000010:1.1:1788572274.959697:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:1.1:1788572274.959722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00010000:00000010:3.1:1788572274.994897:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572274.994916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572275.000803:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:3.1:1788572275.000829:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421d6b40.
00010000:00000010:3.1:1788572275.000836:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572275.006419:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572275.092640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf40.
00000080:00000010:3.1:1788572275.092664:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533da940.
00010000:00000010:3.1:1788572275.101190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572275.101221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572275.113856:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8b60.
00000080:00000010:3.1:1788572275.113874:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342189080.
00010000:00000010:0.1:1788572275.188127:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.188135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:0.1:1788572275.195259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6a0.
00000080:00000010:0.1:1788572275.195288:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534974dac0.
00010000:00000010:0.1:1788572275.209181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.209195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:1.1:1788572275.212843:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f40.
00000080:00000010:1.1:1788572275.212872:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349745ac0.
00010000:00000010:3.1:1788572275.231437:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572275.231891:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572275.238428:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:3.1:1788572275.238457:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519edac0.
00010000:00000010:3.1:1788572275.245098:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572275.245114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572275.252482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8b60.
00000080:00000010:3.1:1788572275.252520:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3b9c0.
00010000:00000010:3.1:1788572275.258684:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572275.258702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572275.268723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8980.
00000080:00000010:3.1:1788572275.268741:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423118c0.
00000080:00000010:0.1:1788572275.391455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:0.1:1788572275.391482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532998c0.
00010000:00000010:0.1:1788572275.396738:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.396754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572275.428630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5920.
00000080:00000010:0.1:1788572275.428656:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519298c0.
00000080:00000010:3.1:1788572275.446723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa58a0.
00000080:00000010:3.1:1788572275.446744:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463ee300.
00010000:00000010:3.1:1788572275.451681:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572275.451707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572275.469795:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8980.
00000080:00000010:3.1:1788572275.469828:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346318840.
00000080:00000010:0.1:1788572275.600704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbe0.
00000080:00000010:0.1:1788572275.600743:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f2100.
00010000:00000010:3.1:1788572275.683744:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572275.683761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572275.690186:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:3.1:1788572275.690215:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535322c200.
00010000:00000010:0.1:1788572275.700514:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.700530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:0.1:1788572275.725813:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8980.
00000080:00000010:0.1:1788572275.725836:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463cb9c0.
00010000:00000010:0.1:1788572275.730451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.730468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572275.743567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:0.1:1788572275.743595:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544ca100.
00010000:00000010:0.1:1788572275.760821:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.760839:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572275.768243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55a0.
00000080:00000010:0.1:1788572275.768269:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352700000.
00010000:00000010:0.1:1788572275.778442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.778457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572275.785701:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:0.1:1788572275.785739:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346203180.
00010000:00000010:0.1:1788572275.798560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.798575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572275.828673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55a0.
00000080:00000010:0.1:1788572275.828694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535268a100.
00010000:00000010:0.1:1788572275.836491:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.836507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572275.854515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:0.1:1788572275.860092:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a1a940.
00010000:00000010:0.1:1788572275.867660:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.867675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572275.880592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55a0.
00000080:00000010:0.1:1788572275.880610:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346044a40.
00010000:00000010:0.1:1788572275.890957:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.890974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:1.1:1788572275.926594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:1.1:1788572275.926616:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351902940.
00010000:00000010:0.1:1788572275.952726:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.952742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:0.1:1788572275.961047:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5620.
00000080:00000010:0.1:1788572275.964505:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953545318c0.
00010000:00000010:0.1:1788572275.970454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572275.970464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572275.983807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5aa0.
00000080:00000010:1.1:1788572275.983828:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351917380.
00010000:00000010:1.1:1788572276.010751:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572276.010767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:1.1:1788572276.016937:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:1.1:1788572276.021625:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a58840.
00010000:00000010:0.1:1788572276.042453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:0.1:1788572276.042477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00010000:00000010:3.1:1788572276.152941:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:3.1:1788572276.152957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572276.161278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:3.1:1788572276.161305:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b13180.
00010000:00000010:3.1:1788572276.173610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572276.173624:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572276.201917:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:3.1:1788572276.201943:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b2d280.
00010000:00000010:0.1:1788572276.318599:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572276.318615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:3.1:1788572276.347439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572276.347458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572276.347790:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:1.1:1788572276.347810:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b1d280.
00000080:00000010:3.1:1788572276.368633:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5760.
00000080:00000010:3.1:1788572276.368664:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494e98c0.
00010000:00000010:3.1:1788572276.382973:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572276.382985:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572276.387571:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:3.1:1788572276.387587:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e2100.
00010000:00000010:3.1:1788572276.480999:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572276.481006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572276.485863:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8a0.
00000080:00000010:3.1:1788572276.485890:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b27380.
00010000:00000010:3.1:1788572276.493453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572276.493470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572276.506440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:3.1:1788572276.506587:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421b8000.
00010000:00000010:3.1:1788572276.584617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572276.584632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572276.592335:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:3.1:1788572276.592362:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342125280.
00010000:00000010:3.1:1788572276.621619:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572276.621627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:1.1:1788572276.688567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572276.688577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:1.1:1788572276.701509:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:1.1:1788572276.701537:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354438000.
00010000:00000010:3.1:1788572276.748674:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572276.748691:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572276.760458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb880.
00000080:00000010:3.1:1788572276.760490:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d1080.
00010000:00000010:3.1:1788572276.778437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572276.778454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572276.804003:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8260.
00000080:00000010:3.1:1788572276.804029:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544b2100.
00010000:00000010:3.1:1788572276.827746:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572276.827763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572276.851434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83a0.
00000080:00000010:3.1:1788572276.851470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495dd280.
00010000:00000010:3.1:1788572276.959490:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572276.959504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572276.966431:0:31:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83a0.
00000080:00000010:3.1:1788572276.966453:0:31:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351909080.
00010000:00000010:1.1:1788572276.970620:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:1.1:1788572276.970634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:1.1:1788572276.980745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8260.
00000080:00000010:1.1:1788572276.980776:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a17380.
00010000:00000010:0.1:1788572277.023539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572277.023554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:1.1:1788572277.031486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbec0.
00000080:00000010:1.1:1788572277.031501:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b01080.
00010000:00000010:3.1:1788572277.128449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572277.128466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572277.136619:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:3.1:1788572277.136649:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b02940.
00010000:00000010:3.1:1788572277.144770:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572277.144787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572277.156958:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83a0.
00000080:00000010:3.1:1788572277.156989:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463bdac0.
00010000:00000010:0.1:1788572277.164612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572277.164627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572277.189462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c40.
00000080:00000010:0.1:1788572277.189496:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b3d280.
00010000:00000010:1.1:1788572277.201981:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:1.1:1788572277.201999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:3.1:1788572277.202192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5220.
00000080:00000010:3.1:1788572277.203767:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534600c200.
00010000:00000010:3.1:1788572277.211895:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572277.211919:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572277.240229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83a0.
00000080:00000010:3.1:1788572277.240247:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534212eb40.
00000080:00000010:1.1:1788572277.289523:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a80840.
00010000:00000010:1.1:1788572277.301438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572277.301448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572277.318441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c55a0.
00000080:00000010:1.1:1788572277.318467:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535322f380.
00010000:00000010:1.1:1788572277.334551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572277.334562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572277.340424:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5700.
00000080:00000010:1.1:1788572277.340649:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463839c0.
00010000:00000010:1.1:1788572277.350427:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572277.350434:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572277.356257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5600.
00000080:00000010:1.1:1788572277.356286:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534234b180.
00010000:00000010:1.1:1788572277.372629:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572277.372643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572277.376958:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5700.
00000080:00000010:1.1:1788572277.376975:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342081080.
00010000:00000010:1.1:1788572277.386072:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572277.386082:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572277.441246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb920.
00000080:00000010:1.1:1788572277.441268:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463c3180.
00010000:00000010:1.1:1788572277.451758:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572277.451772:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:1.1:1788572277.459428:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53c0.
00000080:00000010:1.1:1788572277.459881:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346306300.
00010000:00000010:1.1:1788572277.483216:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572277.483231:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00010000:00000010:0.1:1788572277.536096:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:0.1:1788572277.536108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572277.544429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:0.1:1788572277.546789:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342129080.
00000080:00000010:1.1:1788572277.706750:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba80.
00000080:00000010:1.1:1788572277.706770:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349622940.
00010000:00000010:1.1:1788572277.712449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:1.1:1788572277.712466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:1.1:1788572277.731858:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5980.
00000080:00000010:1.1:1788572277.731892:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346388840.
00000080:00000010:1.1:1788572277.808532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0c0.
00000080:00000010:1.1:1788572277.808563:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346394a40.
00010000:00000010:1.1:1788572277.862440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572277.862457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:1.1:1788572277.874763:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:1.1:1788572277.874779:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534944ca40.
00010000:00000010:1.1:1788572277.879334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572277.879342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572277.890858:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:1.1:1788572277.890877:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b59080.
00010000:00000010:1.1:1788572277.898635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572277.898651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:0.1:1788572277.924659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00000080:00000010:0.1:1788572277.924879:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba5280.
00010000:00000010:1.1:1788572278.036121:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:1.1:1788572278.036137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:0.1:1788572278.048661:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:0.1:1788572278.048679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:1.1:1788572278.064754:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:1.1:1788572278.064777:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422ff380.
00010000:00000010:0.1:1788572278.080780:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572278.080797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:0.1:1788572278.101431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57e0.
00000080:00000010:1.1:1788572278.101480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b0d280.
00000080:00000010:3.1:1788572278.192682:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb060.
00000080:00000010:3.1:1788572278.192719:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534605f380.
00010000:00000010:3.1:1788572278.199556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572278.199572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572278.206545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83a0.
00000080:00000010:3.1:1788572278.206578:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b3b180.
00010000:00000010:3.1:1788572278.229885:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572278.229900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572278.232454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d40.
00000080:00000010:3.1:1788572278.232492:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519b39c0.
00000080:00000010:0.1:1788572278.356855:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:0.1:1788572278.361439:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ba100.
00010000:00000010:1.1:1788572278.368843:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572278.368857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:1.1:1788572278.391488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5680.
00000080:00000010:1.1:1788572278.391513:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527439c0.
00010000:00000010:1.1:1788572278.401122:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572278.401137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572278.433286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59e0.
00000080:00000010:1.1:1788572278.433325:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534958a100.
00000080:00000010:3.1:1788572278.492742:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:3.1:1788572278.492766:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bcdac0.
00010000:00000010:3.1:1788572278.505998:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572278.506016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572278.517436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d40.
00000080:00000010:3.1:1788572278.517460:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b5280.
00010000:00000010:3.1:1788572278.532583:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572278.532595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:0.1:1788572278.538669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8500.
00000080:00000010:0.1:1788572278.538702:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535181a940.
00010000:00000010:1.1:1788572278.553712:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572278.553723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:1.1:1788572278.563560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:1.1:1788572278.567058:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba8840.
00010000:00000010:0.1:1788572278.570579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572278.570648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:0.1:1788572278.594663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f00.
00000080:00000010:0.1:1788572278.594691:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bceb40.
00010000:00000010:1.1:1788572278.605883:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572278.605899:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:1.1:1788572278.630439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56e0.
00000080:00000010:1.1:1788572278.630479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342331080.
00010000:00000010:0.1:1788572278.654731:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:0.1:1788572278.654745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00010000:00000010:3.1:1788572278.675644:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:3.1:1788572278.675657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572278.691426:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:3.1:1788572278.691442:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346332940.
00010000:00000010:3.1:1788572278.696542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572278.696555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572278.705438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d40.
00000080:00000010:3.1:1788572278.713471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346328840.
00010000:00000010:3.1:1788572278.717675:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572278.717690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572278.722745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83e0.
00000080:00000010:3.1:1788572278.722776:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346014a40.
00010000:00000010:3.1:1788572278.738434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572278.738449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572278.749477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d40.
00000080:00000010:3.1:1788572278.749502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b439c0.
00010000:00000010:3.1:1788572278.769546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572278.769557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572278.773825:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83e0.
00000080:00000010:3.1:1788572278.773853:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601d280.
00010000:00000010:3.1:1788572278.833838:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572278.833855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572278.851829:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:3.1:1788572278.851853:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352617380.
00000080:00000010:3.1:1788572278.969379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:3.1:1788572278.969418:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353231080.
00010000:00000010:3.1:1788572279.003642:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572279.003658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572279.015033:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:3.1:1788572279.018718:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519618c0.
00010000:00000010:3.1:1788572279.032435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572279.032450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:0.1:1788572279.055922:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8da0.
00000080:00000010:1.1:1788572279.056156:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495fdac0.
00010000:00000010:0.1:1788572279.066453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572279.066460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:0.1:1788572279.075560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5860.
00000080:00000010:0.1:1788572279.075587:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a739c0.
00010000:00000010:0.1:1788572279.086235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572279.086250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00010000:00000010:3.1:1788572279.266884:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572279.266904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572279.283560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:3.1:1788572279.283583:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535337f380.
00010000:00000010:3.1:1788572279.316555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572279.316570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572279.322554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:3.1:1788572279.322580:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351af5280.
00010000:00000010:3.1:1788572279.360549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572279.360568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572279.360574:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87e0.
00000080:00000010:3.1:1788572279.360591:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352611080.
00010000:00000010:3.1:1788572279.463546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572279.463565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572279.478703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:3.1:1788572279.486655:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460639c0.
00010000:00000010:3.1:1788572279.491422:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572279.491427:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572279.495656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87e0.
00000080:00000010:3.1:1788572279.495695:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463198c0.
00010000:00000010:3.1:1788572279.511420:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572279.511427:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572279.517435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8580.
00010000:00000010:3.1:1788572279.548095:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572279.548106:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572279.551934:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:3.1:1788572279.551959:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533bc200.
00010000:00000010:3.1:1788572279.559139:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572279.559154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572279.572492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8580.
00000080:00000010:3.1:1788572279.572529:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b6b180.
00010000:00000010:3.1:1788572279.650559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572279.650576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572279.674640:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:3.1:1788572279.674683:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354527380.
00010000:00000010:3.1:1788572279.711232:0:2365:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572279.711240:0:2365:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572279.727146:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:3.1:1788572279.727172:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342337380.
00010000:00000010:0.1:1788572279.742784:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:0.1:1788572279.742812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:1.1:1788572279.866503:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572279.866519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:0.1:1788572279.881816:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:0.1:1788572279.881846:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342188840.
00000080:00000010:3.1:1788572279.920460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5500.
00000080:00000010:3.1:1788572279.920487:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533d39c0.
00010000:00000010:3.1:1788572279.943111:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572279.943124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572279.943130:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f60.
00000080:00000010:3.1:1788572279.943146:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953462118c0.
00010000:00000010:3.1:1788572279.958477:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572279.958493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572279.981376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86c0.
00000080:00000010:3.1:1788572279.981415:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534975b9c0.
00010000:00000010:3.1:1788572279.992563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572279.992578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:1.1:1788572279.994044:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f60.
00000080:00000010:1.1:1788572279.994068:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b43180.
00010000:00000010:1.1:1788572280.009623:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572280.009650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:3.1:1788572280.070036:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349662100.
00010000:00000010:3.1:1788572280.083697:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572280.083705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572280.227760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf20.
00000080:00000010:3.1:1788572280.227781:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544c98c0.
00010000:00000010:3.1:1788572280.231723:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572280.231735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572280.235702:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d00.
00000080:00000010:3.1:1788572280.235730:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535263c200.
00010000:00000010:3.1:1788572280.246210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572280.246226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572280.272803:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572280.272830:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346206b40.
00010000:00000010:0.1:1788572280.290523:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:0.1:1788572280.290538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572280.301293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:0.1:1788572280.301318:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544b3180.
00010000:00000010:0.1:1788572280.313294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572280.313323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572280.323553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:0.1:1788572280.323578:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351abca40.
00010000:00000010:0.1:1788572280.335606:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572280.335620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572280.347249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5fa0.
00000080:00000010:0.1:1788572280.347282:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546fe300.
00010000:00000010:0.1:1788572280.353651:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572280.353666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572280.382155:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:0.1:1788572280.382180:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230a100.
00010000:00000010:0.1:1788572280.403639:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572280.403656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572280.426085:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5fa0.
00000080:00000010:0.1:1788572280.433809:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351912940.
00010000:00000010:0.1:1788572280.435882:0:14:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572280.435911:0:14:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572280.444226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:0.1:1788572280.458449:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354534a40.
00010000:00000010:1.1:1788572280.511451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:1.1:1788572280.511470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572280.530453:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354518840.
00010000:00000010:1.1:1788572280.534922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572280.534938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:1.1:1788572280.553551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a00.
00000080:00000010:1.1:1788572280.553582:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349479080.
00010000:00000010:1.1:1788572280.564472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572280.564487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572280.592618:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5480.
00000080:00000010:1.1:1788572280.592631:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420ff380.
00010000:00000010:1.1:1788572280.592641:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572280.592645:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572280.597732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57c0.
00000080:00000010:1.1:1788572280.597752:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b16b40.
00010000:00000010:1.1:1788572280.600440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572280.600536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572280.604542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5240.
00000080:00000010:1.1:1788572280.604572:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351915280.
00010000:00000010:1.1:1788572280.615660:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572280.615686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:1.1:1788572280.627745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57c0.
00000080:00000010:1.1:1788572280.627767:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e3180.
00000080:00000010:3.1:1788572280.685342:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:3.1:1788572280.692439:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460418c0.
00010000:00000010:3.1:1788572280.699672:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572280.699686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572280.729645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572280.729686:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351907380.
00010000:00000010:3.1:1788572280.740352:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572280.740366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00010000:00000010:1.1:1788572280.844203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572280.844216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:1.1:1788572280.844504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:1.1:1788572280.844525:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a59080.
00000080:00000010:3.1:1788572281.003497:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:3.1:1788572281.003513:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b25280.
00010000:00000010:3.1:1788572281.013959:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572281.014799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572281.033521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572281.037509:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e6300.
00000080:00000010:3.1:1788572281.102797:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:3.1:1788572281.105702:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b7b180.
00010000:00000010:3.1:1788572281.126868:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572281.126882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572281.130720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572281.137728:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497a5280.
00010000:00000010:3.1:1788572281.152745:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572281.152758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572281.159828:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8400.
00000080:00000010:3.1:1788572281.159849:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ace300.
00010000:00000010:1.1:1788572281.163711:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.163723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00010000:00000010:3.1:1788572281.205596:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572281.205610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572281.219570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf20.
00000080:00000010:3.1:1788572281.219597:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532298c0.
00010000:00000010:1.1:1788572281.262454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.262469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:1.1:1788572281.268038:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8400.
00000080:00000010:1.1:1788572281.268061:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494fdac0.
00010000:00000010:3.1:1788572281.307802:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572281.307819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:1.1:1788572281.322463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:1.1:1788572281.331894:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a6300.
00010000:00000010:1.1:1788572281.357649:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.357674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572281.368587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5800.
00000080:00000010:1.1:1788572281.368611:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534214eb40.
00010000:00000010:1.1:1788572281.401436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.401454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572281.419469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ca0.
00000080:00000010:1.1:1788572281.419500:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544b18c0.
00010000:00000010:1.1:1788572281.456838:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.456854:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572281.488685:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5800.
00000080:00000010:1.1:1788572281.488714:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d39c0.
00010000:00000010:1.1:1788572281.505417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.505430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572281.526192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ca0.
00000080:00000010:0.1:1788572281.526278:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463ceb40.
00010000:00000010:3.1:1788572281.561123:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:3.1:1788572281.561138:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572281.577163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:3.1:1788572281.577194:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495db9c0.
00010000:00000010:3.1:1788572281.585678:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572281.585687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572281.585692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8660.
00000080:00000010:3.1:1788572281.589725:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352707380.
00010000:00000010:3.1:1788572281.596720:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572281.596735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572281.614035:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00000080:00000010:1.1:1788572281.616746:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535263f380.
00010000:00000010:1.1:1788572281.631087:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:1.1:1788572281.631099:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:3.1:1788572281.752442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:3.1:1788572281.752468:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a5d280.
00010000:00000010:3.1:1788572281.760556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572281.760571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572281.771893:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00000080:00000010:3.1:1788572281.771913:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ad2100.
00000080:00000010:1.1:1788572281.853611:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:1.1:1788572281.853641:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ad7380.
00010000:00000010:3.1:1788572281.858546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572281.858565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:3.1:1788572281.863434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50c0.
00000080:00000010:3.1:1788572281.863458:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534606dac0.
00010000:00000010:1.1:1788572281.871436:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.871605:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:1.1:1788572281.876437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00000080:00000010:1.1:1788572281.876471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b5eb40.
00010000:00000010:1.1:1788572281.880439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.880458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572281.887595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58a0.
00000080:00000010:1.1:1788572281.891575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421a39c0.
00010000:00000010:1.1:1788572281.896060:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.896066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572281.901894:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53a0.
00000080:00000010:1.1:1788572281.901921:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534633a940.
00010000:00000010:1.1:1788572281.909535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.909613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:0.1:1788572281.929821:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58a0.
00000080:00000010:0.1:1788572281.929853:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349748840.
00010000:00000010:1.1:1788572281.935606:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572281.935616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:1.1:1788572281.945477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa57a0.
00000080:00000010:1.1:1788572281.945497:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421e5ac0.
00010000:00000010:0.1:1788572282.013567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:0.1:1788572282.013581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:0.1:1788572282.022844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:0.1:1788572282.022880:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535199d280.
00010000:00000010:0.1:1788572282.032361:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572282.032378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572282.041892:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b20.
00000080:00000010:0.1:1788572282.041922:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354470000.
00010000:00000010:0.1:1788572282.049579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572282.049592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572282.061440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53a0.
00000080:00000010:0.1:1788572282.061468:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342332940.
00000080:00000010:0.1:1788572282.275459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:0.1:1788572282.275482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534218e300.
00010000:00000010:3.1:1788572282.348481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572282.348501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572282.348508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:3.1:1788572282.354637:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351990000.
00010000:00000010:3.1:1788572282.358600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572282.358620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572282.361694:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8700.
00000080:00000010:3.1:1788572282.361728:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535197b9c0.
00010000:00000010:3.1:1788572282.369795:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572282.369809:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00010000:00000010:0.1:1788572282.473661:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572282.473679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:0.1:1788572282.473685:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb060.
00000080:00000010:0.1:1788572282.473702:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf1080.
00010000:00000010:0.1:1788572282.473710:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572282.473714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572282.477993:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa57c0.
00000080:00000010:0.1:1788572282.478011:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544718c0.
00010000:00000010:0.1:1788572282.494010:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572282.494023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:1.1:1788572282.495870:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5740.
00000080:00000010:1.1:1788572282.495892:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535269f380.
00010000:00000010:3.1:1788572282.516526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572282.516541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:3.1:1788572282.524876:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5200.
00000080:00000010:3.1:1788572282.524905:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bbe300.
00010000:00000010:3.1:1788572282.538419:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572282.538450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572282.554311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8700.
00000080:00000010:3.1:1788572282.554330:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535329ca40.
00010000:00000010:3.1:1788572282.644569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572282.644585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572282.657445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:3.1:1788572282.657469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494b39c0.
00010000:00000010:3.1:1788572282.665484:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572282.665498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00010000:00000010:3.1:1788572282.699537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572282.699552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572282.707994:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:3.1:1788572282.708013:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353378000.
00010000:00000010:3.1:1788572282.719742:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572282.719761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572282.724700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c00.
00000080:00000010:3.1:1788572282.724735:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349542940.
00010000:00000010:3.1:1788572282.728595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572282.728619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572282.757219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572282.757317:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353230840.
00010000:00000010:0.1:1788572282.855444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572282.855457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:1.1:1788572282.866421:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0c0.
00000080:00000010:1.1:1788572282.870485:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535455a100.
00010000:00000010:1.1:1788572282.880800:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572282.880806:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572282.898572:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5640.
00000080:00000010:1.1:1788572282.898599:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544a18c0.
00010000:00000010:3.1:1788572282.969781:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572282.969791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572282.969797:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba80.
00000080:00000010:3.1:1788572282.969812:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534621b9c0.
00010000:00000010:3.1:1788572282.969820:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572282.969823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00010000:00000010:0.1:1788572283.004745:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:0.1:1788572283.004759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:0.1:1788572283.017776:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:0.1:1788572283.017784:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:0.1:1788572283.030440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:0.1:1788572283.030471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534638b9c0.
00010000:00000010:0.1:1788572283.034443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572283.034452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572283.043422:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5880.
00000080:00000010:0.1:1788572283.043437:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519dca40.
00010000:00000010:0.1:1788572283.047431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572283.047446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572283.055434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:0.1:1788572283.055498:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544c9080.
00010000:00000010:0.1:1788572283.055570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572283.055575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572283.061848:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5fc0.
00000080:00000010:0.1:1788572283.061864:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534944b180.
00010000:00000010:0.1:1788572283.061874:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572283.061878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572283.066491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5880.
00000080:00000010:0.1:1788572283.066518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534639a940.
00010000:00000010:0.1:1788572283.071628:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572283.071636:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572283.076765:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:0.1:1788572283.076788:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463dd280.
00010000:00000010:0.1:1788572283.087851:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572283.087856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572283.100616:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5880.
00000080:00000010:0.1:1788572283.100636:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d0000.
00010000:00000010:0.1:1788572283.114553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572283.114568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572283.119625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:0.1:1788572283.119655:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534210c200.
00000080:00000010:3.1:1788572283.163744:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:3.1:1788572283.163769:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342346b40.
00000080:00000010:0.1:1788572283.175622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:0.1:1788572283.175664:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463c2940.
00010000:00000010:1.1:1788572283.188434:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:1.1:1788572283.188449:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00010000:00000010:1.1:1788572283.241445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572283.241460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:1.1:1788572283.252429:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb920.
00000080:00000010:1.1:1788572283.252456:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534212dac0.
00010000:00000010:0.1:1788572283.262939:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572283.262946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:0.1:1788572283.278435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:0.1:1788572283.278461:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a8dac0.
00010000:00000010:0.1:1788572283.296241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572283.296255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572283.346762:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f60.
00000080:00000010:1.1:1788572283.350451:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be0840.
00010000:00000010:1.1:1788572283.364328:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572283.364346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00010000:00000010:3.1:1788572283.413554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572283.413568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572283.425562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb880.
00000080:00000010:3.1:1788572283.425584:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352689080.
00010000:00000010:3.1:1788572283.432676:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572283.432690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572283.446555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:3.1:1788572283.446583:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349597380.
00010000:00000010:3.1:1788572283.458452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572283.458468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572283.470778:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8720.
00000080:00000010:3.1:1788572283.470803:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f8840.
00010000:00000010:3.1:1788572283.477458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572283.477474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572283.488440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:3.1:1788572283.488469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953496218c0.
00010000:00000010:3.1:1788572283.496586:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572283.496599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:1.1:1788572283.507566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8720.
00000080:00000010:1.1:1788572283.507593:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423498c0.
00010000:00000010:3.1:1788572283.521112:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572283.521129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00010000:00000010:3.1:1788572283.596499:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572283.596511:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572283.618713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbec0.
00000080:00000010:0.1:1788572283.622788:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349634a40.
00010000:00000010:0.1:1788572283.702956:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572283.702972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:0.1:1788572283.844533:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e18c0.
00010000:00000010:1.1:1788572283.859009:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:1.1:1788572283.859026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:3.1:1788572284.045431:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8a0.
00000080:00000010:3.1:1788572284.045448:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535454a100.
00010000:00000010:3.1:1788572284.050555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572284.050572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572284.065447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:3.1:1788572284.065471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535454c200.
00010000:00000010:3.1:1788572284.075550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572284.075567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572284.085926:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572284.085953:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519db9c0.
00010000:00000010:3.1:1788572284.093619:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572284.093645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:3.1:1788572284.440512:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:3.1:1788572284.440533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572284.456962:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:3.1:1788572284.456987:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533d0840.
00010000:00000010:3.1:1788572284.490040:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572284.490052:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572284.490057:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572284.490072:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346212100.
00010000:00000010:0.1:1788572284.497484:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:0.1:1788572284.497496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:1.1:1788572284.506306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c00.
00000080:00000010:1.1:1788572284.506332:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534952a940.
00010000:00000010:1.1:1788572284.515453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572284.515464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00010000:00000010:3.1:1788572284.586582:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572284.586590:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572284.593112:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572284.593139:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aa1080.
00010000:00000010:3.1:1788572284.593151:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572284.593158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572284.607896:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8700.
00000080:00000010:3.1:1788572284.615598:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526aa100.
00010000:00000010:3.1:1788572284.618694:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572284.618702:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572284.628811:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572284.628837:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526ab180.
00010000:00000010:0.1:1788572284.701355:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:0.1:1788572284.701373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:0.1:1788572284.724897:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:0.1:1788572284.724924:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a7eb40.
00010000:00000010:1.1:1788572284.746994:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:1.1:1788572284.747012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00010000:00000010:3.1:1788572284.885862:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572284.885883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572284.921166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:3.1:1788572284.921197:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495fc200.
00010000:00000010:0.1:1788572284.948428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:0.1:1788572284.948444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:1.1:1788572284.996258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:1.1:1788572284.996280:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a6eb40.
00010000:00000010:0.1:1788572285.012503:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572285.012516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:0.1:1788572285.020897:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57e0.
00000080:00000010:0.1:1788572285.020926:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534631f380.
00010000:00000010:0.1:1788572285.036441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572285.036457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572285.074404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5260.
00000080:00000010:0.1:1788572285.074438:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463a98c0.
00010000:00000010:3.1:1788572285.197614:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572285.197645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572285.218549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:3.1:1788572285.218575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533adac0.
00010000:00000010:3.1:1788572285.236438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572285.236449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572285.259587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d89a0.
00000080:00000010:3.1:1788572285.259611:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518f6b40.
00010000:00000010:3.1:1788572285.359444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572285.359461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572285.371575:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:3.1:1788572285.371589:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518f1080.
00010000:00000010:3.1:1788572285.461482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572285.461500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572285.482389:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbe0.
00000080:00000010:3.1:1788572285.482536:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519ba940.
00010000:00000010:3.1:1788572285.494042:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572285.494052:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572285.512956:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8660.
00000080:00000010:3.1:1788572285.512980:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535270d280.
00000080:00000010:0.1:1788572285.590280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:0.1:1788572285.590314:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601b180.
00010000:00000010:0.1:1788572285.590326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572285.590331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572285.591586:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e40.
00000080:00000010:0.1:1788572285.591605:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354480840.
00010000:00000010:0.1:1788572285.642487:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:0.1:1788572285.642503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:1.1:1788572285.642555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:1.1:1788572285.642575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534621e300.
00010000:00000010:0.1:1788572285.647922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572285.647930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:0.1:1788572285.654897:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f00.
00000080:00000010:0.1:1788572285.654914:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346321080.
00010000:00000010:0.1:1788572285.676215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572285.676237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572285.676243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e80.
00000080:00000010:0.1:1788572285.676261:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534214b9c0.
00010000:00000010:0.1:1788572285.683241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572285.683257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572285.683262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5460.
00000080:00000010:0.1:1788572285.683277:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346202940.
00010000:00000010:0.1:1788572285.693722:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572285.693731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572285.699172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e80.
00000080:00000010:0.1:1788572285.699189:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d8000.
00010000:00000010:1.1:1788572285.718442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572285.718450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00010000:00000010:0.1:1788572285.784996:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:0.1:1788572285.785011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:1.1:1788572285.797468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:1.1:1788572285.797500:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342346300.
00010000:00000010:0.1:1788572285.806482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572285.806490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:0.1:1788572285.817389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5660.
00000080:00000010:0.1:1788572285.817436:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535263e300.
00010000:00000010:1.1:1788572285.826471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:1.1:1788572285.826487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00010000:00000010:3.1:1788572285.899560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572285.899573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:3.1:1788572286.066792:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:3.1:1788572286.066808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572286.075041:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6a0.
00000080:00000010:3.1:1788572286.075071:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e4200.
00010000:00000010:3.1:1788572286.350604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572286.350631:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572286.374842:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:3.1:1788572286.374869:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497aa100.
00010000:00000010:3.1:1788572286.396834:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572286.396849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572286.401732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572286.401759:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aceb40.
00010000:00000010:3.1:1788572286.419633:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572286.419652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572286.456464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8760.
00000080:00000010:3.1:1788572286.456494:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534604c200.
00000080:00000010:3.1:1788572286.663841:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:3.1:1788572286.667640:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a52940.
00000080:00000010:0.1:1788572286.790436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf40.
00000080:00000010:0.1:1788572286.790462:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947ca40.
00010000:00000010:1.1:1788572286.822451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572286.822465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:1.1:1788572286.859715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5860.
00000080:00000010:1.1:1788572286.859778:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b139c0.
00010000:00000010:3.1:1788572286.948536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:3.1:1788572286.948549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572286.973548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb960.
00000080:00000010:3.1:1788572286.973572:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351920000.
00010000:00000010:1.1:1788572286.996458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572286.996474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572287.004528:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d00.
00000080:00000010:1.1:1788572287.008458:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346045ac0.
00010000:00000010:1.1:1788572287.024441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572287.024457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572287.032432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5300.
00000080:00000010:1.1:1788572287.032820:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460498c0.
00010000:00000010:1.1:1788572287.066528:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572287.066542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:1.1:1788572287.083438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:1.1:1788572287.083467:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346392100.
00010000:00000010:1.1:1788572287.091454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572287.091471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572287.100714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54a0.
00000080:00000010:1.1:1788572287.104450:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527018c0.
00010000:00000010:1.1:1788572287.117437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572287.117451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572287.125437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57a0.
00000080:00000010:1.1:1788572287.125458:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953462039c0.
00010000:00000010:1.1:1788572287.135442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572287.135456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572287.144439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54a0.
00000080:00000010:1.1:1788572287.144463:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be39c0.
00010000:00000010:1.1:1788572287.156530:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572287.156545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572287.167487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57a0.
00000080:00000010:1.1:1788572287.167513:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544beb40.
00010000:00000010:1.1:1788572287.186535:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572287.186542:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572287.191562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54a0.
00000080:00000010:1.1:1788572287.191589:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342360000.
00000080:00000010:3.1:1788572287.277507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb180.
00000080:00000010:3.1:1788572287.277534:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544fb9c0.
00010000:00000010:3.1:1788572287.288207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572287.288228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572287.297654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86c0.
00000080:00000010:3.1:1788572287.301544:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423639c0.
00010000:00000010:3.1:1788572287.325767:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572287.325776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572287.325782:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f60.
00000080:00000010:3.1:1788572287.325795:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351abb180.
00010000:00000010:3.1:1788572287.332984:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572287.332997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:1.1:1788572287.355363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86c0.
00000080:00000010:1.1:1788572287.355378:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ab0000.
00010000:00000010:1.1:1788572287.360316:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572287.360333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572287.360338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5620.
00000080:00000010:1.1:1788572287.360354:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463118c0.
00010000:00000010:1.1:1788572287.397432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572287.397452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00010000:00000010:0.1:1788572287.407844:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572287.407855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:0.1:1788572287.413429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbaa0.
00000080:00000010:0.1:1788572287.413461:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b76b40.
00010000:00000010:0.1:1788572287.420728:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572287.420742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572287.426459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5d20.
00000080:00000010:0.1:1788572287.426485:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353291080.
00010000:00000010:0.1:1788572287.430561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572287.430570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:0.1:1788572287.434670:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ac0.
00000080:00000010:0.1:1788572287.434694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342136b40.
00010000:00000010:0.1:1788572287.441572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572287.441585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572287.455624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5d20.
00000080:00000010:0.1:1788572287.455651:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354732100.
00010000:00000010:3.1:1788572287.569564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572287.569580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572287.582965:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfa0.
00000080:00000010:3.1:1788572287.582988:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a398c0.
00010000:00000010:1.1:1788572287.744569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572287.744577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:1.1:1788572287.752542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb160.
00000080:00000010:1.1:1788572287.752565:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b4a940.
00010000:00000010:1.1:1788572287.756432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572287.756525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572287.767903:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59e0.
00000080:00000010:1.1:1788572287.770732:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349754200.
00010000:00000010:1.1:1788572287.781814:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572287.781824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572287.786592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f60.
00000080:00000010:1.1:1788572287.786629:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534633e300.
00010000:00000010:1.1:1788572287.804654:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572287.804661:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:3.1:1788572287.810410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59e0.
00000080:00000010:3.1:1788572287.810455:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533dd280.
00010000:00000010:3.1:1788572287.820434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572287.820460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572287.835091:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d80.
00000080:00000010:1.1:1788572287.839817:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342182940.
00010000:00000010:0.1:1788572287.849680:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572287.849690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:0.1:1788572287.856373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5b20.
00000080:00000010:0.1:1788572287.856421:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346326300.
00010000:00000010:3.1:1788572288.027445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572288.027460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572288.045080:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4a0.
00000080:00000010:3.1:1788572288.045109:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352696300.
00010000:00000010:3.1:1788572288.078810:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572288.078822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572288.093361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d80.
00000080:00000010:3.1:1788572288.093404:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342082940.
00010000:00000010:3.1:1788572288.107449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572288.107466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572288.147417:0:1046:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8580.
00000080:00000010:3.1:1788572288.147454:0:1046:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519998c0.
00000080:00000010:1.1:1788572288.251140:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1c0.
00000080:00000010:1.1:1788572288.251180:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535198c200.
00010000:00000010:1.1:1788572288.271436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572288.271447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00010000:00000010:3.1:1788572288.318695:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:3.1:1788572288.318709:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572288.334848:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3a0.
00000080:00000010:3.1:1788572288.334865:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421d2100.
00010000:00000010:3.1:1788572288.420672:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572288.420689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572288.442782:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00000080:00000010:3.1:1788572288.442799:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b48000.
00010000:00000010:3.1:1788572288.453662:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572288.453675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:1.1:1788572288.485657:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342180000.
00010000:00000010:1.1:1788572288.494665:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572288.494679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572288.499448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50e0.
00000080:00000010:1.1:1788572288.499468:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421ac200.
00010000:00000010:1.1:1788572288.504786:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572288.504802:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572288.513621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00000080:00000010:1.1:1788572288.513649:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463298c0.
00010000:00000010:1.1:1788572288.521427:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572288.521703:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:1.1:1788572288.525711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50e0.
00000080:00000010:1.1:1788572288.530787:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519798c0.
00000080:00000010:3.1:1788572288.570856:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00000080:00000010:3.1:1788572288.580559:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535441eb40.
00010000:00000010:3.1:1788572288.591218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572288.591235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572288.597650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:3.1:1788572288.597671:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354478000.
00010000:00000010:3.1:1788572288.597682:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572288.597687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00010000:00000010:0.1:1788572288.725347:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572288.725362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572288.732772:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:0.1:1788572288.732788:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352692940.
00010000:00000010:0.1:1788572288.740114:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572288.740126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00010000:00000010:1.1:1788572288.768772:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:1.1:1788572288.768780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572288.769691:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa51c0.
00000080:00000010:0.1:1788572288.769709:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535327f380.
00010000:00000010:0.1:1788572288.798870:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572288.798887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:1.1:1788572288.818051:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb560.
00000080:00000010:1.1:1788572288.818076:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d0840.
00000080:00000010:1.1:1788572288.898618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:1.1:1788572288.898645:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351929080.
00010000:00000010:3.1:1788572288.995653:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572288.995669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572289.004491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb000.
00000080:00000010:3.1:1788572289.004519:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463a39c0.
00010000:00000010:3.1:1788572289.022852:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572289.022860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572289.026922:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:3.1:1788572289.026949:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a539c0.
00010000:00000010:0.1:1788572289.103638:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:0.1:1788572289.103652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:1.1:1788572289.116821:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe00.
00000080:00000010:1.1:1788572289.116838:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354502100.
00010000:00000010:1.1:1788572289.125797:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572289.125803:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:1.1:1788572289.141559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c80.
00000080:00000010:1.1:1788572289.141576:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534215d280.
00010000:00000010:1.1:1788572289.141587:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572289.141694:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:1.1:1788572289.146883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d80.
00000080:00000010:1.1:1788572289.146912:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534623ca40.
00010000:00000010:1.1:1788572289.162739:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572289.162754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572289.185478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58e0.
00000080:00000010:1.1:1788572289.185507:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463418c0.
00010000:00000010:0.1:1788572289.212738:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:0.1:1788572289.212754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:3.1:1788572289.260578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba40.
00000080:00000010:3.1:1788572289.260611:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349544a40.
00010000:00000010:1.1:1788572289.279467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:1.1:1788572289.279482:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:0.1:1788572289.306949:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:0.1:1788572289.311475:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420b98c0.
00010000:00000010:0.1:1788572289.330492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572289.330508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572289.355471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa51a0.
00000080:00000010:0.1:1788572289.355503:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463eb9c0.
00000080:00000010:3.1:1788572289.391751:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b70000.
00010000:00000010:3.1:1788572289.428112:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572289.428132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572289.433289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d40.
00000080:00000010:3.1:1788572289.433317:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535269e300.
00000080:00000010:1.1:1788572289.593433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb300.
00000080:00000010:1.1:1788572289.593738:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346029080.
00010000:00000010:1.1:1788572289.607445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572289.607462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572289.627650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00000080:00000010:1.1:1788572289.627678:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3ca40.
00010000:00000010:1.1:1788572289.654209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572289.654223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572289.671761:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00000080:00000010:1.1:1788572289.672852:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519ee300.
00010000:00000010:0.1:1788572289.686765:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:0.1:1788572289.686779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572289.711452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00000080:00000010:1.1:1788572289.711485:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534952eb40.
00010000:00000010:3.1:1788572289.724658:0:1046:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572289.724674:0:1046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572289.737424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5580.
00000080:00000010:3.1:1788572289.737458:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535440ca40.
00010000:00000010:1.1:1788572289.874612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:1.1:1788572289.874629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00010000:00000010:3.1:1788572289.913656:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572289.913672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572289.956874:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:3.1:1788572289.956897:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349756b40.
00010000:00000010:1.1:1788572289.994453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572289.994468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:1.1:1788572290.018779:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83a0.
00000080:00000010:1.1:1788572290.018804:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b4b180.
00010000:00000010:0.1:1788572290.043991:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:0.1:1788572290.044007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00010000:00000010:3.1:1788572290.113526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572290.113543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572290.132923:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd60.
00000080:00000010:3.1:1788572290.132950:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535327dac0.
00010000:00000010:3.1:1788572290.155528:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572290.155546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572290.173467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d60.
00000080:00000010:3.1:1788572290.173514:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535198f380.
00010000:00000010:3.1:1788572290.198682:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572290.198697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:1.1:1788572290.283680:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352758840.
00010000:00000010:1.1:1788572290.305278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572290.305291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572290.313833:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5bc0.
00000080:00000010:1.1:1788572290.318476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d2940.
00010000:00000010:1.1:1788572290.349005:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572290.349020:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572290.367685:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59e0.
00000080:00000010:1.1:1788572290.367718:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535447d280.
00010000:00000010:0.1:1788572290.499685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572290.499700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:1.1:1788572290.516075:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf80.
00000080:00000010:1.1:1788572290.516107:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342185280.
00010000:00000010:0.1:1788572290.539830:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572290.539841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:0.1:1788572290.545457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f40.
00000080:00000010:0.1:1788572290.545486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534632b9c0.
00010000:00000010:1.1:1788572290.559255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:1.1:1788572290.559273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572290.666302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfc0.
00000080:00000010:0.1:1788572290.666329:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342236300.
00010000:00000010:0.1:1788572290.678764:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572290.678773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:1.1:1788572290.684493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5800.
00000080:00000010:1.1:1788572290.684522:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535448dac0.
00000080:00000010:3.1:1788572290.710621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54c0.
00000080:00000010:3.1:1788572290.710646:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349751080.
00000080:00000010:3.1:1788572290.776315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8e0.
00000080:00000010:3.1:1788572290.781228:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535275d280.
00010000:00000010:3.1:1788572290.807072:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572290.807090:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572290.820436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8260.
00000080:00000010:3.1:1788572290.820466:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352759080.
00010000:00000010:3.1:1788572290.838158:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572290.838173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572290.848423:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:0.1:1788572291.051504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3e0.
00000080:00000010:0.1:1788572291.051533:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352701080.
00010000:00000010:1.1:1788572291.060661:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572291.060678:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:1.1:1788572291.084450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56a0.
00000080:00000010:0.1:1788572291.192885:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb680.
00000080:00000010:0.1:1788572291.192913:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353298000.
00010000:00000010:0.1:1788572291.201156:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572291.201184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:0.1:1788572291.220420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5920.
00000080:00000010:0.1:1788572291.220440:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463139c0.
00010000:00000010:3.1:1788572291.246426:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572291.246432:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:3.1:1788572291.257263:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa51e0.
00000080:00000010:3.1:1788572291.257290:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463a8000.
00010000:00000010:3.1:1788572291.279439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572291.279454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572291.279459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:3.1:1788572291.279474:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b9dac0.
00000080:00000010:1.1:1788572291.326455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8260.
00000080:00000010:1.1:1788572291.326481:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349529080.
00010000:00000010:1.1:1788572291.359448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572291.359464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00010000:00000010:0.1:1788572291.500435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572291.500453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00010000:00000010:0.1:1788572291.565745:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:0.1:1788572291.565768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:0.1:1788572291.569466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb420.
00000080:00000010:0.1:1788572291.569503:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494fd280.
00010000:00000010:0.1:1788572291.569515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572291.569521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572291.585489:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5000.
00000080:00000010:0.1:1788572291.585524:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519a18c0.
00010000:00000010:0.1:1788572291.601554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572291.601577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572291.609471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a20.
00000080:00000010:0.1:1788572291.609499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352517380.
00010000:00000010:0.1:1788572291.626433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572291.626447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572291.643602:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5000.
00000080:00000010:0.1:1788572291.643619:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342340000.
00010000:00000010:0.1:1788572291.647579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572291.647595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00010000:00000010:3.1:1788572291.773734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572291.773750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572291.781450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb620.
00000080:00000010:3.1:1788572291.781479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d2940.
00010000:00000010:3.1:1788572291.786925:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572291.786940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572291.817461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00010000:00000010:1.1:1788572291.826750:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572291.826764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:0.1:1788572291.881165:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:0.1:1788572291.881183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:0.1:1788572291.897418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8c0.
00000080:00000010:0.1:1788572291.902464:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535453b9c0.
00010000:00000010:0.1:1788572291.906193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572291.906209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572291.933590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5700.
00000080:00000010:0.1:1788572291.933616:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460298c0.
00010000:00000010:0.1:1788572291.933622:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572291.933626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:3.1:1788572291.999504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb640.
00000080:00000010:3.1:1788572292.003473:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b28840.
00010000:00000010:3.1:1788572292.003492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572292.003498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572292.015719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:3.1:1788572292.015744:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b1f380.
00010000:00000010:3.1:1788572292.035010:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572292.035027:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572292.045999:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:3.1:1788572292.046030:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535258dac0.
00010000:00000010:1.1:1788572292.063935:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572292.063950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572292.079102:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:1.1:1788572292.079126:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a45ac0.
00010000:00000010:0.1:1788572292.096040:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:0.1:1788572292.096052:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00010000:00000010:3.1:1788572292.369795:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572292.369809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572292.369815:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb60.
00000080:00000010:3.1:1788572292.369832:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533f6300.
00010000:00000010:3.1:1788572292.391769:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572292.391784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572292.403456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:3.1:1788572292.403484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354532940.
00000080:00000010:3.1:1788572292.511425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd00.
00000080:00000010:3.1:1788572292.513747:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351925280.
00010000:00000010:3.1:1788572292.532850:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572292.532863:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572292.532868:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:3.1:1788572292.532882:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497af380.
00010000:00000010:0.1:1788572292.555450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572292.555464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:3.1:1788572292.674624:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572292.674632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572292.683721:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5e0.
00000080:00000010:3.1:1788572292.683741:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495aa940.
00010000:00000010:0.1:1788572292.693997:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:0.1:1788572292.694013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:0.1:1788572292.698727:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:0.1:1788572292.698741:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349666b40.
00010000:00000010:0.1:1788572292.709624:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572292.709637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572292.719490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa54a0.
00000080:00000010:0.1:1788572292.719519:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544ba940.
00010000:00000010:0.1:1788572292.720610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572292.720617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00010000:00000010:0.1:1788572292.767143:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572292.767150:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572292.772698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0a0.
00000080:00000010:0.1:1788572292.772728:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346202100.
00010000:00000010:1.1:1788572292.826477:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572292.826493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:1.1:1788572292.855430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb740.
00000080:00000010:1.1:1788572292.855451:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349679080.
00010000:00000010:1.1:1788572292.863330:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572292.863346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:3.1:1788572292.996452:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b0000.
00010000:00000010:3.1:1788572293.007459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572293.007473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572293.017500:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8980.
00000080:00000010:3.1:1788572293.021815:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519dc200.
00010000:00000010:3.1:1788572293.025623:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572293.025638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572293.036487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8160.
00000080:00000010:3.1:1788572293.036522:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354542940.
00000080:00000010:0.1:1788572293.109976:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535454a940.
00010000:00000010:1.1:1788572293.123917:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572293.123926:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:1.1:1788572293.144441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f60.
00000080:00000010:1.1:1788572293.144476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519b2940.
00010000:00000010:1.1:1788572293.162441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572293.162458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00000080:00000010:1.1:1788572293.180480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5de0.
00000080:00000010:1.1:1788572293.180504:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518f4200.
00010000:00000010:1.1:1788572293.202285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572293.202300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00000080:00000010:0.1:1788572293.220901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5e60.
00000080:00000010:0.1:1788572293.220924:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346215280.
00010000:00000010:3.1:1788572293.341096:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:3.1:1788572293.341114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572293.355619:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb480.
00000080:00000010:3.1:1788572293.355647:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497539c0.
00010000:00000010:3.1:1788572293.426981:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572293.427003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572293.438973:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb140.
00000080:00000010:3.1:1788572293.443054:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352616b40.
00000080:00000010:3.1:1788572293.546457:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb20.
00000080:00000010:3.1:1788572293.546488:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346034200.
00010000:00000010:3.1:1788572293.553136:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572293.553164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572293.579255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d88c0.
00000080:00000010:3.1:1788572293.579277:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535330b9c0.
00010000:00000010:3.1:1788572293.584844:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572293.584860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572293.638088:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd80.
00000080:00000010:3.1:1788572293.638122:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346062940.
00010000:00000010:0.1:1788572293.661733:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:0.1:1788572293.661752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:0.1:1788572293.675568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d88c0.
00000080:00000010:0.1:1788572293.677829:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e18c0.
00010000:00000010:0.1:1788572293.727317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572293.727343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572293.762987:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b80.
00000080:00000010:3.1:1788572293.830655:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc80.
00000080:00000010:3.1:1788572293.830678:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495298c0.
00010000:00000010:3.1:1788572293.857979:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572293.857995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572293.858001:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8be0.
00000080:00000010:3.1:1788572293.858016:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546ce300.
00010000:00000010:3.1:1788572293.862434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572293.862461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572293.880944:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f00.
00000080:00000010:3.1:1788572293.880966:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546ca100.
00010000:00000010:3.1:1788572293.920681:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572293.920691:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:3.1:1788572293.948719:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572293.948735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00010000:00000010:1.1:1788572293.971455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:1.1:1788572293.971471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:0.1:1788572294.083431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:0.1:1788572294.083454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:1.1:1788572294.094442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc00.
00000080:00000010:1.1:1788572294.108897:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518f18c0.
00010000:00000010:1.1:1788572294.128913:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572294.128928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:1.1:1788572294.135990:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53a0.
00000080:00000010:1.1:1788572294.136016:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb1080.
00010000:00000010:1.1:1788572294.147570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572294.147585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:1.1:1788572294.160352:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58a0.
00000080:00000010:1.1:1788572294.160379:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bc4200.
00010000:00000010:3.1:1788572294.217467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:3.1:1788572294.217481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572294.237767:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb080.
00000080:00000010:3.1:1788572294.237793:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354549080.
00010000:00000010:0.1:1788572294.241515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572294.241526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572294.497704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb800.
00000080:00000010:3.1:1788572294.497735:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535190c200.
00010000:00000010:3.1:1788572294.526471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572294.526495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572294.894919:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb940.
00000080:00000010:3.1:1788572294.894940:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534621a100.
00010000:00000010:3.1:1788572294.903441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572294.903458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572294.917441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8e60.
00000080:00000010:3.1:1788572294.917470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342123180.
00010000:00000010:3.1:1788572295.081038:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572295.081056:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572295.219989:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbda0.
00000080:00000010:3.1:1788572295.220021:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422deb40.
00010000:00000010:3.1:1788572295.340073:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572295.340088:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572295.379835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4e0.
00000080:00000010:3.1:1788572295.379867:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534639e300.
00000080:00000010:0.1:1788572295.435358:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346388000.
00010000:00000010:3.1:1788572295.489448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572295.489463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572295.508628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd40.
00000080:00000010:3.1:1788572295.508660:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a8b9c0.
00010000:00000010:1.1:1788572295.677467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572295.677483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572295.713610:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9a0.
00000080:00000010:1.1:1788572295.713649:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba0000.
00010000:00000010:1.1:1788572295.733670:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572295.733689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572295.746727:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00000080:00000010:1.1:1788572295.749204:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494939c0.
00010000:00000010:3.1:1788572295.803475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:3.1:1788572295.803492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572295.803498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb040.
00000080:00000010:3.1:1788572295.803515:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354612940.
00010000:00000010:3.1:1788572295.821452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572295.821467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:3.1:1788572295.943505:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572295.943514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572295.955759:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe80.
00000080:00000010:3.1:1788572295.955778:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534234b9c0.
00000080:00000010:0.1:1788572296.001617:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349448000.
00010000:00000010:3.1:1788572296.058662:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572296.058684:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572296.066597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1e0.
00000080:00000010:3.1:1788572296.066622:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342083180.
00010000:00000010:3.1:1788572296.081567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572296.081583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572296.138449:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8880.
00000080:00000010:3.1:1788572296.138476:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be4a40.
00010000:00000010:3.1:1788572296.138486:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572296.138492:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572296.156441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d85c0.
00000080:00000010:3.1:1788572296.156467:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a11080.
00010000:00000010:3.1:1788572296.312541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572296.312560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572296.323449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfe0.
00000080:00000010:3.1:1788572296.323480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463b9080.
00000080:00000010:3.1:1788572296.370829:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d85c0.
00000080:00000010:3.1:1788572296.370861:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953496598c0.
00010000:00000010:3.1:1788572296.378448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572296.378469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572296.397448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8880.
00000080:00000010:3.1:1788572296.397479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534605dac0.
00010000:00000010:3.1:1788572296.436522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572296.436543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572296.447720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbcc0.
00000080:00000010:3.1:1788572296.451712:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349566300.
00010000:00000010:3.1:1788572296.456657:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572296.456666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:1.1:1788572296.514482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8880.
00000080:00000010:1.1:1788572296.514527:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474eb40.
00010000:00000010:1.1:1788572296.520054:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572296.520073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00000080:00000010:1.1:1788572296.528906:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50a0.
00000080:00000010:1.1:1788572296.532485:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535455ca40.
00010000:00000010:3.1:1788572296.558432:0:1106:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572296.558456:0:1106:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:3.1:1788572296.583391:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56a0.
00000080:00000010:3.1:1788572296.583771:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b0c200.
00010000:00000010:1.1:1788572296.599493:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572296.599510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:1.1:1788572296.606709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00000080:00000010:1.1:1788572296.622570:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a9eb40.
00000080:00000010:3.1:1788572296.808494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb80.
00000080:00000010:3.1:1788572296.816330:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534623dac0.
00010000:00000010:3.1:1788572296.871453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:3.1:1788572296.871472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572296.892455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb760.
00000080:00000010:3.1:1788572296.892490:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352569080.
00010000:00000010:3.1:1788572296.963663:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572296.963670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572297.085215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbca0.
00000080:00000010:3.1:1788572297.085252:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342235280.
00010000:00000010:3.1:1788572297.089461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572297.089479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572297.103488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:3.1:1788572297.103518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421e3180.
00010000:00000010:3.1:1788572297.118769:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572297.118786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572297.132578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8280.
00000080:00000010:3.1:1788572297.132611:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346337380.
00010000:00000010:3.1:1788572297.146494:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572297.146509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572297.162478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:3.1:1788572297.166004:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421aca40.
00010000:00000010:3.1:1788572297.177500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572297.177520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00010000:00000010:3.1:1788572297.232127:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572297.232146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:0.1:1788572297.372516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7c0.
00000080:00000010:0.1:1788572297.376254:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352754a40.
00010000:00000010:1.1:1788572297.387471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:1.1:1788572297.387489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572297.409883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5120.
00000080:00000010:0.1:1788572297.409904:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544d6300.
00010000:00000010:1.1:1788572297.426522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572297.426536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572297.447896:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53a0.
00000080:00000010:1.1:1788572297.447929:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495dc200.
00010000:00000010:1.1:1788572297.447941:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572297.447946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572297.468922:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f60.
00000080:00000010:1.1:1788572297.468950:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351990840.
00010000:00000010:1.1:1788572297.480784:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572297.480803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:0.1:1788572297.660854:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb540.
00000080:00000010:0.1:1788572297.660872:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533d98c0.
00010000:00000010:1.1:1788572297.670685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572297.670700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572297.679810:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5da0.
00000080:00000010:0.1:1788572297.679839:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352705ac0.
00000080:00000010:0.1:1788572297.730895:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56e0.
00000080:00000010:0.1:1788572297.730923:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bfd280.
00010000:00000010:1.1:1788572297.734713:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572297.734729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:1.1:1788572297.743669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5da0.
00000080:00000010:1.1:1788572297.747540:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a1080.
00010000:00000010:1.1:1788572297.768342:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572297.768358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572297.787131:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5fc0.
00000080:00000010:1.1:1788572297.787165:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518d4a40.
00010000:00000010:1.1:1788572297.806682:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572297.806699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572297.825795:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c40.
00000080:00000010:1.1:1788572297.825835:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495df380.
00010000:00000010:1.1:1788572297.836541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572297.836556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:1.1:1788572297.950962:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:1.1:1788572297.950981:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:1.1:1788572297.964968:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1a0.
00000080:00000010:1.1:1788572297.965000:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351972100.
00010000:00000010:1.1:1788572297.977787:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572297.977813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572298.004983:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5240.
00000080:00000010:1.1:1788572298.005018:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d2100.
00010000:00000010:1.1:1788572298.015849:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572298.015867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572298.041162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57c0.
00000080:00000010:3.1:1788572298.213678:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353233180.
00010000:00000010:3.1:1788572298.230734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572298.230742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572298.235763:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:3.1:1788572298.235793:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349541080.
00010000:00000010:3.1:1788572298.250527:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572298.250547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572298.258664:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572298.263375:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342311080.
00010000:00000010:3.1:1788572298.289152:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572298.289170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572298.314389:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:3.1:1788572298.314447:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534634b180.
00010000:00000010:0.1:1788572298.376439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:0.1:1788572298.376465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:3.1:1788572298.443500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:3.1:1788572298.443524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572298.450648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7e0.
00000080:00000010:3.1:1788572298.458150:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346224a40.
00010000:00000010:3.1:1788572298.538230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572298.538243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572298.544369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc40.
00000080:00000010:3.1:1788572298.544414:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544cf380.
00010000:00000010:0.1:1788572298.565789:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572298.565804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00010000:00000010:3.1:1788572298.623733:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572298.623751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572298.632601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb860.
00000080:00000010:3.1:1788572298.635505:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544ed280.
00010000:00000010:1.1:1788572298.654455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572298.654472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:1.1:1788572298.669736:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fa0.
00000080:00000010:1.1:1788572298.669766:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a2940.
00010000:00000010:0.1:1788572298.715969:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:0.1:1788572298.715979:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:0.1:1788572298.723480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbba0.
00000080:00000010:0.1:1788572298.723512:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953496639c0.
00010000:00000010:0.1:1788572298.741474:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572298.741483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00010000:00000010:3.1:1788572298.913653:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572298.913682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572298.932740:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2c0.
00000080:00000010:3.1:1788572298.932769:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b1a100.
00010000:00000010:3.1:1788572298.944935:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572298.944952:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572298.956492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8540.
00000080:00000010:3.1:1788572298.961477:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351912100.
00010000:00000010:3.1:1788572298.971771:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572298.971790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572298.988807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fa0.
00000080:00000010:3.1:1788572298.988833:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e0000.
00000080:00000010:0.1:1788572299.057785:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947dac0.
00010000:00000010:1.1:1788572299.071118:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572299.071135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:1.1:1788572299.082578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5000.
00000080:00000010:1.1:1788572299.082611:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535451eb40.
00010000:00000010:1.1:1788572299.088766:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:1.1:1788572299.088783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572299.099870:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5580.
00000080:00000010:1.1:1788572299.101635:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a5e300.
00010000:00000010:3.1:1788572299.188460:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572299.188474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572299.208901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbde0.
00000080:00000010:3.1:1788572299.208932:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351916b40.
00000080:00000010:0.1:1788572299.277708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe20.
00000080:00000010:0.1:1788572299.289644:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351acdac0.
00010000:00000010:1.1:1788572299.294545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:1.1:1788572299.294560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:3.1:1788572299.378370:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb40.
00000080:00000010:3.1:1788572299.378426:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a1e300.
00010000:00000010:3.1:1788572299.384447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572299.384463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572299.393440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fc0.
00000080:00000010:3.1:1788572299.393472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e4a40.
00010000:00000010:0.1:1788572299.420632:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:0.1:1788572299.420649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:0.1:1788572299.420655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87c0.
00000080:00000010:0.1:1788572299.420672:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421bb180.
00000080:00000010:3.1:1788572299.492447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb240.
00000080:00000010:3.1:1788572299.492584:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535258eb40.
00010000:00000010:3.1:1788572299.496570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:3.1:1788572299.496578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:1.1:1788572299.526554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0e0.
00000080:00000010:1.1:1788572299.526589:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e7380.
00010000:00000010:1.1:1788572299.533612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572299.533627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572299.547476:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5080.
00000080:00000010:1.1:1788572299.547506:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354519080.
00010000:00000010:1.1:1788572299.555597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:1.1:1788572299.555616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572299.579474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f20.
00000080:00000010:1.1:1788572299.579517:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534222c200.
00010000:00000010:0.1:1788572299.595691:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:0.1:1788572299.595704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:3.1:1788572299.744788:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7a0.
00000080:00000010:3.1:1788572299.744814:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535263ca40.
00010000:00000010:0.1:1788572299.768572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572299.768593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572299.772856:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5940.
00000080:00000010:0.1:1788572299.772882:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352706300.
00010000:00000010:0.1:1788572299.900853:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572299.900861:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:0.1:1788572299.913253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb100.
00000080:00000010:0.1:1788572299.913280:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534217b9c0.
00010000:00000010:0.1:1788572299.918844:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572299.918866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572299.941566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5760.
00000080:00000010:0.1:1788572299.941593:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463cd280.
00000080:00000010:3.1:1788572299.951895:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f20.
00000080:00000010:3.1:1788572299.951921:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534214a940.
00010000:00000010:3.1:1788572299.971054:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572299.971070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572299.978408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:3.1:1788572299.978433:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535443b180.
00010000:00000010:3.1:1788572300.029807:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572300.029828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572300.037546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb340.
00000080:00000010:3.1:1788572300.037575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494feb40.
00010000:00000010:0.1:1788572300.054434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:0.1:1788572300.054443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:0.1:1788572300.066725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d20.
00000080:00000010:0.1:1788572300.066752:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533eb9c0.
00010000:00000010:3.1:1788572300.127616:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:3.1:1788572300.127623:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572300.137749:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5a0.
00000080:00000010:3.1:1788572300.137772:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342309080.
00000080:00000010:3.1:1788572300.225590:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519b18c0.
00010000:00000010:3.1:1788572300.235799:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572300.236115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572300.334655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2e0.
00000080:00000010:0.1:1788572300.335451:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535181a100.
00010000:00000010:3.1:1788572300.424868:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:3.1:1788572300.424883:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572300.431440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbc0.
00000080:00000010:3.1:1788572300.435693:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518d18c0.
00000080:00000010:1.1:1788572300.469676:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb00.
00000080:00000010:1.1:1788572300.469701:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519bb9c0.
00010000:00000010:1.1:1788572300.484296:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572300.484308:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572300.501845:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5840.
00000080:00000010:1.1:1788572300.501861:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535193e300.
00010000:00000010:1.1:1788572300.510895:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572300.510907:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572300.531863:0:2354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5640.
00000080:00000010:1.1:1788572300.531890:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534975ca40.
00010000:00000010:0.1:1788572300.576435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572300.576457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:0.1:1788572300.584586:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb280.
00000080:00000010:0.1:1788572300.584617:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346012100.
00010000:00000010:0.1:1788572300.584626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572300.584632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572300.589466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a40.
00000080:00000010:0.1:1788572300.589545:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534223a100.
00000080:00000010:3.1:1788572300.686436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb360.
00000080:00000010:3.1:1788572300.686464:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342317380.
00010000:00000010:0.1:1788572300.695184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572300.695200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572300.701581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00000080:00000010:3.1:1788572300.701673:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535256ca40.
00010000:00000010:3.1:1788572300.711458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572300.711477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:0.1:1788572300.888627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb360.
00000080:00000010:0.1:1788572300.888644:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535256e300.
00010000:00000010:1.1:1788572300.899595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572300.899609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:1.1:1788572300.903983:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5800.
00000080:00000010:1.1:1788572300.904003:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495fd280.
00010000:00000010:1.1:1788572300.911603:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572300.911617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:1.1:1788572300.919462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53e0.
00000080:00000010:1.1:1788572300.919488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953462298c0.
00010000:00000010:1.1:1788572300.927806:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572300.927822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:0.1:1788572300.937764:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5120.
00000080:00000010:0.1:1788572300.941793:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352612100.
00010000:00000010:3.1:1788572301.045689:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572301.045698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572301.045704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf00.
00000080:00000010:3.1:1788572301.045719:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346065ac0.
00010000:00000010:3.1:1788572301.053731:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572301.053740:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572301.061430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00000080:00000010:3.1:1788572301.061460:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463ed280.
00010000:00000010:0.1:1788572301.071904:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572301.071913:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:0.1:1788572301.097720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d85c0.
00000080:00000010:0.1:1788572301.097752:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354544200.
00010000:00000010:0.1:1788572301.097760:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572301.097765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572301.097769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5720.
00000080:00000010:0.1:1788572301.097777:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf0840.
00010000:00000010:0.1:1788572301.105893:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.105905:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572301.114514:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56a0.
00000080:00000010:0.1:1788572301.114536:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b6e300.
00010000:00000010:0.1:1788572301.123044:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.123052:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572301.131858:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5720.
00000080:00000010:0.1:1788572301.131881:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354536300.
00010000:00000010:0.1:1788572301.136835:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.136849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572301.151531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56a0.
00000080:00000010:0.1:1788572301.151561:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535269b180.
00010000:00000010:0.1:1788572301.155720:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.155746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572301.163066:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5720.
00000080:00000010:0.1:1788572301.163095:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bcc200.
00010000:00000010:0.1:1788572301.187227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.187243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572301.190161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56a0.
00000080:00000010:0.1:1788572301.190178:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527518c0.
00010000:00000010:3.1:1788572301.283779:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:3.1:1788572301.283798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572301.310457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb280.
00000080:00000010:3.1:1788572301.310488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352740000.
00010000:00000010:3.1:1788572301.318723:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572301.318751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572301.332576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00000080:00000010:3.1:1788572301.332607:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b87380.
00010000:00000010:0.1:1788572301.403854:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:0.1:1788572301.403869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:0.1:1788572301.422431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb460.
00000080:00000010:0.1:1788572301.422459:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534975dac0.
00010000:00000010:0.1:1788572301.575463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.575480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572301.585681:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb00.
00000080:00000010:0.1:1788572301.585710:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a198c0.
00010000:00000010:1.1:1788572301.600592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572301.600600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572301.613434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5de0.
00000080:00000010:1.1:1788572301.613467:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351903180.
00010000:00000010:1.1:1788572301.617081:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572301.617089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572301.620578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50c0.
00000080:00000010:1.1:1788572301.620603:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b38000.
00010000:00000010:1.1:1788572301.633599:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572301.633615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:1.1:1788572301.641655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5900.
00000080:00000010:1.1:1788572301.644820:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b018c0.
00010000:00000010:1.1:1788572301.666854:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572301.666863:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:1.1:1788572301.666868:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50c0.
00000080:00000010:1.1:1788572301.666882:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463bf380.
00010000:00000010:3.1:1788572301.666980:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572301.666986:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:3.1:1788572301.684062:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5900.
00000080:00000010:3.1:1788572301.684089:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422fa100.
00010000:00000010:3.1:1788572301.689166:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572301.689183:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:0.1:1788572301.692860:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8360.
00000080:00000010:0.1:1788572301.692884:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354494200.
00010000:00000010:0.1:1788572301.698567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.698583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572301.705474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53a0.
00000080:00000010:0.1:1788572301.710914:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535451b9c0.
00010000:00000010:0.1:1788572301.721648:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.721662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572301.729477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5200.
00000080:00000010:0.1:1788572301.732602:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349566b40.
00010000:00000010:0.1:1788572301.737460:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.737477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572301.755198:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53a0.
00000080:00000010:0.1:1788572301.755223:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474a100.
00010000:00000010:0.1:1788572301.766403:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572301.766411:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:1.1:1788572301.772635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5200.
00000080:00000010:1.1:1788572301.772661:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533c6300.
00010000:00000010:3.1:1788572301.882511:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572301.882532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572301.892926:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2e0.
00000080:00000010:3.1:1788572301.896480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342128840.
00010000:00000010:3.1:1788572301.906438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572301.906455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572301.921709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f40.
00000080:00000010:3.1:1788572301.921738:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535268eb40.
00010000:00000010:3.1:1788572301.933761:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572301.933781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572301.945876:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8640.
00000080:00000010:3.1:1788572301.945906:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527a1080.
00010000:00000010:3.1:1788572302.030439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572302.030460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572302.042332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5a0.
00000080:00000010:3.1:1788572302.042370:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463c5ac0.
00010000:00000010:3.1:1788572302.047626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572302.047639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572302.051629:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8640.
00000080:00000010:3.1:1788572302.051656:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349636300.
00010000:00000010:1.1:1788572302.067652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572302.067669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:1.1:1788572302.085451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d81e0.
00000080:00000010:1.1:1788572302.085470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a9080.
00010000:00000010:0.1:1788572302.094637:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572302.094643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:3.1:1788572302.131482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb340.
00000080:00000010:3.1:1788572302.131518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463818c0.
00010000:00000010:3.1:1788572302.146079:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572302.146096:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572302.153675:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8640.
00000080:00000010:3.1:1788572302.153706:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a98c0.
00010000:00000010:3.1:1788572302.160569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572302.160585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572302.175486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8140.
00000080:00000010:3.1:1788572302.175515:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349591080.
00010000:00000010:3.1:1788572302.229957:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572302.229972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:3.1:1788572302.271424:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572302.271430:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572302.277473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb100.
00000080:00000010:3.1:1788572302.277499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d8840.
00010000:00000010:1.1:1788572302.314685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:1.1:1788572302.314711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:0.1:1788572302.328886:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:0.1:1788572302.328902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:1.1:1788572302.335530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7a0.
00000080:00000010:1.1:1788572302.341249:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b5a100.
00010000:00000010:0.1:1788572302.349778:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572302.349787:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00000080:00000010:0.1:1788572302.364553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5e60.
00000080:00000010:0.1:1788572302.374476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527a6b40.
00000080:00000010:3.1:1788572302.617525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbc0.
00000080:00000010:3.1:1788572302.617559:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474c200.
00010000:00000010:0.1:1788572302.632711:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:0.1:1788572302.632729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:0.1:1788572302.657475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8e60.
00000080:00000010:0.1:1788572302.657504:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b03180.
00010000:00000010:0.1:1788572302.674455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572302.674470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572302.698888:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5740.
00000080:00000010:0.1:1788572302.698906:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354492100.
00010000:00000010:0.1:1788572302.808461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572302.808475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:1.1:1788572302.823635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0e0.
00000080:00000010:1.1:1788572302.827544:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534218ca40.
00010000:00000010:0.1:1788572302.831637:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572302.831655:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:0.1:1788572302.847732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c56e0.
00000080:00000010:0.1:1788572302.847758:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b42940.
00010000:00000010:0.1:1788572302.871802:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572302.871818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572302.879630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5640.
00000080:00000010:0.1:1788572302.879654:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497598c0.
00010000:00000010:0.1:1788572302.889527:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572302.889541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572302.894435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5840.
00000080:00000010:0.1:1788572302.894470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519bd280.
00010000:00000010:0.1:1788572302.901742:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572302.901760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572302.915793:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5640.
00000080:00000010:0.1:1788572302.915818:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535193b180.
00010000:00000010:0.1:1788572302.932239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572302.932249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572302.935722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5840.
00000080:00000010:0.1:1788572302.935749:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354524a40.
00010000:00000010:0.1:1788572302.948431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572302.948451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572302.966700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5640.
00000080:00000010:0.1:1788572302.966729:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d4a40.
00010000:00000010:0.1:1788572302.966738:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572302.966743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572302.976315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5840.
00000080:00000010:0.1:1788572302.985607:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533bb180.
00010000:00000010:0.1:1788572302.998437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572302.998458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:3.1:1788572303.129254:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb240.
00000080:00000010:3.1:1788572303.129285:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354545ac0.
00010000:00000010:3.1:1788572303.129298:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572303.129303:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572303.165504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb40.
00000080:00000010:3.1:1788572303.165536:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352697380.
00000080:00000010:3.1:1788572303.201053:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572303.201072:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460618c0.
00010000:00000010:3.1:1788572303.204567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572303.204582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572303.221894:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8be0.
00000080:00000010:3.1:1788572303.221916:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e2940.
00000080:00000010:3.1:1788572303.242712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572303.242736:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352610000.
00010000:00000010:3.1:1788572303.247563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572303.247578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572303.264594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8be0.
00000080:00000010:3.1:1788572303.284293:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346228840.
00010000:00000010:3.1:1788572303.291681:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572303.291695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572303.295680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572303.295699:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346223180.
00010000:00000010:3.1:1788572303.295708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572303.295712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00010000:00000010:0.1:1788572303.382453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:0.1:1788572303.382465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572303.409061:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbde0.
00000080:00000010:3.1:1788572303.409081:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351967380.
00010000:00000010:3.1:1788572303.421569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572303.421585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572303.425796:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572303.425825:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353230000.
00010000:00000010:0.1:1788572303.483866:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:0.1:1788572303.483882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:0.1:1788572303.533920:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572303.533936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572303.592879:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe20.
00000080:00000010:3.1:1788572303.592905:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346333180.
00010000:00000010:3.1:1788572303.596925:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572303.596941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572303.607695:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572303.607719:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421e98c0.
00010000:00000010:3.1:1788572303.626344:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572303.626362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572303.635450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8100.
00000080:00000010:3.1:1788572303.635469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601dac0.
00010000:00000010:3.1:1788572303.643702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572303.643712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572303.658126:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572303.658150:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544818c0.
00000080:00000010:0.1:1788572303.789824:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2c0.
00000080:00000010:0.1:1788572303.789851:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535181ca40.
00010000:00000010:0.1:1788572303.800137:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572303.800152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572303.808752:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5020.
00000080:00000010:0.1:1788572303.816204:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354533180.
00010000:00000010:0.1:1788572303.833329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572303.833350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572303.842468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ba0.
00000080:00000010:0.1:1788572303.842488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342330000.
00010000:00000010:0.1:1788572303.842506:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572303.842512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00010000:00000010:0.1:1788572303.899236:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:0.1:1788572303.899251:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:0.1:1788572303.912730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbba0.
00000080:00000010:0.1:1788572303.912758:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349478840.
00010000:00000010:0.1:1788572303.912766:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572303.912771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572303.922692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5160.
00000080:00000010:0.1:1788572303.928241:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463ba940.
00010000:00000010:1.1:1788572303.938927:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:1.1:1788572303.938939:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572303.949649:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5460.
00000080:00000010:0.1:1788572303.949677:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534605ca40.
00010000:00000010:0.1:1788572303.955440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:0.1:1788572303.955447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:3.1:1788572303.974782:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb860.
00000080:00000010:3.1:1788572303.974818:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b318c0.
00000080:00000010:3.1:1788572304.047485:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7e0.
00000080:00000010:3.1:1788572304.047506:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a1ca40.
00010000:00000010:3.1:1788572304.066025:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572304.066033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:3.1:1788572304.108459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572304.108469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572304.113123:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1a0.
00000080:00000010:3.1:1788572304.118445:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495939c0.
00010000:00000010:3.1:1788572304.157678:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:3.1:1788572304.157696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572304.181896:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb540.
00000080:00000010:3.1:1788572304.181921:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495ab9c0.
00000080:00000010:0.1:1788572304.229264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7c0.
00000080:00000010:0.1:1788572304.229296:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d5280.
00010000:00000010:1.1:1788572304.234456:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:1.1:1788572304.234464:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00010000:00000010:3.1:1788572304.269443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572304.269458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572304.288620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbca0.
00000080:00000010:3.1:1788572304.292676:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532d1080.
00000080:00000010:0.1:1788572304.379545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb80.
00000080:00000010:0.1:1788572304.379568:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346394200.
00010000:00000010:1.1:1788572304.382993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:1.1:1788572304.383010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:3.1:1788572304.400784:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a40.
00000080:00000010:3.1:1788572304.400830:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527a6300.
00000080:00000010:0.1:1788572304.485763:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb760.
00000080:00000010:0.1:1788572304.485784:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353244a40.
00010000:00000010:1.1:1788572304.490461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572304.490611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:1.1:1788572304.524639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55c0.
00000080:00000010:1.1:1788572304.524676:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421a2100.
00010000:00000010:1.1:1788572304.528984:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572304.528998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:0.1:1788572304.553007:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53a0.
00000080:00000010:0.1:1788572304.553031:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230f380.
00010000:00000010:3.1:1788572304.560889:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572304.560918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:3.1:1788572304.574688:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c00.
00000080:00000010:3.1:1788572304.579336:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495ab180.
00010000:00000010:3.1:1788572304.591444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572304.591461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:3.1:1788572304.609803:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fe0.
00000080:00000010:3.1:1788572304.609837:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544e5280.
00000080:00000010:0.1:1788572304.650434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:0.1:1788572304.654539:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342080000.
00010000:00000010:3.1:1788572304.685179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572304.685196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:0.1:1788572304.763737:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:0.1:1788572304.763755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:3.1:1788572304.784808:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:3.1:1788572304.784835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572304.794701:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfe0.
00000080:00000010:3.1:1788572304.797727:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463f8000.
00000080:00000010:3.1:1788572304.917415:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1e0.
00000080:00000010:3.1:1788572304.917592:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526898c0.
00010000:00000010:3.1:1788572304.935234:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572304.935246:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00010000:00000010:3.1:1788572305.013724:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572305.013743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572305.020928:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbcc0.
00000080:00000010:3.1:1788572305.020955:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534212f380.
00010000:00000010:0.1:1788572305.032430:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572305.032459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:0.1:1788572305.048900:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d60.
00000080:00000010:0.1:1788572305.048928:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546fb9c0.
00010000:00000010:0.1:1788572305.149661:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572305.149676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:3.1:1788572305.211214:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572305.211221:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572305.221748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe80.
00000080:00000010:3.1:1788572305.221769:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535455c200.
00010000:00000010:0.1:1788572305.442458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:0.1:1788572305.442476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:1.1:1788572305.534827:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb040.
00000080:00000010:1.1:1788572305.534857:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534623a100.
00010000:00000010:3.1:1788572305.546434:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572305.546442:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:3.1:1788572305.551454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c52a0.
00000080:00000010:3.1:1788572305.551479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463ec200.
00000080:00000010:3.1:1788572305.776647:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d0000.
00010000:00000010:3.1:1788572305.792103:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572305.792126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:3.1:1788572305.806583:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d40.
00000080:00000010:3.1:1788572305.806613:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351993180.
00010000:00000010:1.1:1788572305.884590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572305.884605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:1.1:1788572305.900448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9a0.
00000080:00000010:1.1:1788572305.900652:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535269a100.
00010000:00000010:1.1:1788572305.914433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572305.914450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572305.935311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5540.
00000080:00000010:1.1:1788572305.935336:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d4200.
00010000:00000010:1.1:1788572305.949924:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572305.949950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572305.959108:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5320.
00000080:00000010:1.1:1788572305.959139:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf9080.
00010000:00000010:1.1:1788572305.965617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572305.965647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572305.997523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5540.
00000080:00000010:1.1:1788572305.997541:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342188000.
00010000:00000010:1.1:1788572306.008439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572306.008459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572306.008743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5320.
00000080:00000010:1.1:1788572306.008758:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421aeb40.
00010000:00000010:0.1:1788572306.271229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:0.1:1788572306.271247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:1.1:1788572306.300922:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd40.
00000080:00000010:1.1:1788572306.301058:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a6300.
00010000:00000010:1.1:1788572306.304718:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:1.1:1788572306.304735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00010000:00000010:3.1:1788572306.324640:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572306.324657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572306.336116:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4e0.
00000080:00000010:3.1:1788572306.336147:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534958ca40.
00010000:00000010:3.1:1788572306.355281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572306.355297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572306.360745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:3.1:1788572306.365577:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bff380.
00010000:00000010:3.1:1788572306.382420:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572306.382676:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:1.1:1788572306.422451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbda0.
00000080:00000010:1.1:1788572306.422491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354473180.
00010000:00000010:0.1:1788572306.439774:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572306.439790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:0.1:1788572306.458679:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f80.
00000080:00000010:0.1:1788572306.458703:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354470840.
00010000:00000010:0.1:1788572306.483470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572306.483497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572306.497447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b00.
00000080:00000010:0.1:1788572306.497484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba5ac0.
00010000:00000010:0.1:1788572306.511682:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572306.511698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00010000:00000010:1.1:1788572306.595909:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572306.595923:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:1.1:1788572306.603526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb800.
00000080:00000010:1.1:1788572306.603560:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b58840.
00010000:00000010:1.1:1788572306.614729:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572306.614742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572306.641424:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5e40.
00000080:00000010:1.1:1788572306.641441:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421aa100.
00010000:00000010:1.1:1788572306.654463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572306.654490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:0.1:1788572306.734066:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb940.
00000080:00000010:0.1:1788572306.734098:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b57380.
00010000:00000010:1.1:1788572306.787178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:1.1:1788572306.787186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:0.1:1788572306.823914:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb080.
00000080:00000010:0.1:1788572306.824022:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519ea940.
00010000:00000010:3.1:1788572306.875847:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572306.875862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572306.884437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc00.
00000080:00000010:3.1:1788572306.892794:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494b6b40.
00010000:00000010:3.1:1788572306.906352:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572306.906365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572306.921133:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:3.1:1788572306.921160:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534215eb40.
00010000:00000010:3.1:1788572306.955311:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572306.955325:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572306.965304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8da0.
00000080:00000010:3.1:1788572306.965334:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534623c200.
00010000:00000010:3.1:1788572306.993903:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572306.993921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572306.997899:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:3.1:1788572306.997928:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534606d280.
00010000:00000010:3.1:1788572307.036697:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572307.036714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572307.085341:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8da0.
00000080:00000010:3.1:1788572307.085367:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420be300.
00010000:00000010:3.1:1788572307.092085:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572307.092101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:0.1:1788572307.122578:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a5dac0.
00000080:00000010:3.1:1788572307.122643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00010000:00000010:0.1:1788572307.149458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572307.149467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572307.157160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5520.
00000080:00000010:0.1:1788572307.157194:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ad1080.
00000080:00000010:0.1:1788572307.237777:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc80.
00000080:00000010:0.1:1788572307.237795:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353293180.
00010000:00000010:0.1:1788572307.260196:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572307.260205:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572307.264531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a60.
00000080:00000010:0.1:1788572307.264556:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463a0840.
00010000:00000010:0.1:1788572307.270098:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572307.270106:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572307.275115:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5400.
00000080:00000010:0.1:1788572307.278780:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346316b40.
00010000:00000010:0.1:1788572307.299506:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572307.299526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00010000:00000010:1.1:1788572307.437612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572307.437628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:3.1:1788572307.483564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd80.
00000080:00000010:3.1:1788572307.483600:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ad2940.
00010000:00000010:0.1:1788572307.497047:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572307.497055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:0.1:1788572307.597434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb20.
00000080:00000010:0.1:1788572307.597465:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346226b40.
00010000:00000010:3.1:1788572307.630827:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572307.630842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572307.635543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb140.
00000080:00000010:3.1:1788572307.635563:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544b39c0.
00010000:00000010:3.1:1788572307.680467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572307.680485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572307.689751:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb480.
00000080:00000010:3.1:1788572307.698562:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ac8000.
00010000:00000010:3.1:1788572307.706809:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572307.706817:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572307.711669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8da0.
00000080:00000010:3.1:1788572307.711703:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b78840.
00000080:00000010:3.1:1788572307.862616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb740.
00000080:00000010:3.1:1788572307.862643:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421b9080.
00010000:00000010:3.1:1788572307.899447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572307.899465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572307.912447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8da0.
00000080:00000010:3.1:1788572307.912483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b24a40.
00010000:00000010:3.1:1788572307.931710:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572307.931729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572307.942901:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8580.
00000080:00000010:3.1:1788572307.942947:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a4ca40.
00010000:00000010:0.1:1788572308.257381:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572308.257409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:1.1:1788572308.282451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5e0.
00000080:00000010:1.1:1788572308.282479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535451e300.
00010000:00000010:3.1:1788572308.344511:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572308.344538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572308.358553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd00.
00000080:00000010:3.1:1788572308.362809:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ab5280.
00010000:00000010:3.1:1788572308.375763:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572308.375781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00010000:00000010:1.1:1788572308.415458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:1.1:1788572308.415474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:0.1:1788572308.419470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0a0.
00000080:00000010:0.1:1788572308.419499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533e9080.
00010000:00000010:1.1:1788572308.427572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572308.427588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:1.1:1788572308.456241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5500.
00000080:00000010:1.1:1788572308.456270:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f3180.
00010000:00000010:0.1:1788572308.585456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:0.1:1788572308.585471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:0.1:1788572308.595621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb60.
00000080:00000010:0.1:1788572308.595645:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f39c0.
00010000:00000010:0.1:1788572308.607523:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572308.607537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572308.643881:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5460.
00000080:00000010:0.1:1788572308.643897:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a5280.
00010000:00000010:1.1:1788572308.644059:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:1.1:1788572308.644066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572308.648293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:0.1:1788572308.648320:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346222100.
00010000:00000010:1.1:1788572308.654589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572308.654605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:1.1:1788572308.662613:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e80.
00000080:00000010:1.1:1788572308.662626:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535268d280.
00010000:00000010:1.1:1788572308.667767:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572308.667784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572308.675439:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f60.
00000080:00000010:1.1:1788572308.677649:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bd18c0.
00010000:00000010:1.1:1788572308.681810:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572308.681827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572308.691864:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5b20.
00000080:00000010:1.1:1788572308.691888:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463ca100.
00010000:00000010:1.1:1788572308.701451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572308.701466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572308.721625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f60.
00000080:00000010:1.1:1788572308.721643:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ef380.
00000080:00000010:3.1:1788572308.747545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5b20.
00000080:00000010:3.1:1788572308.747565:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421ec200.
00010000:00000010:3.1:1788572308.755752:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572308.755768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572308.760598:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d80.
00000080:00000010:3.1:1788572308.760617:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354484a40.
00010000:00000010:3.1:1788572308.764653:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572308.764671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572308.774550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f60.
00000080:00000010:3.1:1788572308.774577:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349742940.
00010000:00000010:0.1:1788572308.787557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:0.1:1788572308.787574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:1.1:1788572308.843459:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460139c0.
00010000:00000010:0.1:1788572308.856430:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572308.856439:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:0.1:1788572308.877552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50c0.
00000080:00000010:0.1:1788572308.877578:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346332100.
00010000:00000010:0.1:1788572308.885605:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572308.885612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572308.885617:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ee0.
00000080:00000010:0.1:1788572308.885630:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba98c0.
00010000:00000010:1.1:1788572308.895415:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572308.895427:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:1.1:1788572308.909650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f00.
00000080:00000010:1.1:1788572308.909668:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bc5ac0.
00010000:00000010:1.1:1788572308.914214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572308.914224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:1.1:1788572308.934140:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5000.
00000080:00000010:1.1:1788572308.934157:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518f0840.
00010000:00000010:1.1:1788572308.934162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572308.934166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:0.1:1788572309.024840:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8c0.
00000080:00000010:0.1:1788572309.024866:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf2100.
00000080:00000010:3.1:1788572309.085854:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb620.
00000080:00000010:3.1:1788572309.085871:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ba940.
00010000:00000010:3.1:1788572309.094518:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572309.094525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00010000:00000010:0.1:1788572309.129298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:0.1:1788572309.129307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572309.153180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb640.
00000080:00000010:0.1:1788572309.153211:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351819080.
00010000:00000010:1.1:1788572309.170039:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.170048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:1.1:1788572309.174447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5340.
00000080:00000010:1.1:1788572309.174474:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535270a100.
00000080:00000010:0.1:1788572309.312762:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a6a940.
00010000:00000010:1.1:1788572309.328575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.328592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:1.1:1788572309.332745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5cc0.
00000080:00000010:1.1:1788572309.332775:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a78840.
00010000:00000010:1.1:1788572309.345509:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.345523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572309.351697:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5700.
00000080:00000010:1.1:1788572309.351734:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342134200.
00010000:00000010:1.1:1788572309.361695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.361712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572309.377708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ca0.
00000080:00000010:1.1:1788572309.377809:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546cca40.
00010000:00000010:1.1:1788572309.377817:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.377822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572309.387546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5700.
00000080:00000010:1.1:1788572309.387584:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535330d280.
00010000:00000010:1.1:1788572309.394684:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.394698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:1.1:1788572309.410548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ca0.
00000080:00000010:1.1:1788572309.410628:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352613180.
00000080:00000010:1.1:1788572309.431271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5700.
00000080:00000010:1.1:1788572309.431299:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b96b40.
00010000:00000010:0.1:1788572309.432026:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572309.432035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00010000:00000010:0.1:1788572309.435600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572309.435612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:0.1:1788572309.439455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5600.
00000080:00000010:0.1:1788572309.442930:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534606b180.
00010000:00000010:0.1:1788572309.446696:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572309.446712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572309.455549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5220.
00000080:00000010:0.1:1788572309.455583:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e5ac0.
00010000:00000010:1.1:1788572309.467414:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:1.1:1788572309.467430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:3.1:1788572309.502470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3e0.
00000080:00000010:3.1:1788572309.502501:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546c9080.
00010000:00000010:3.1:1788572309.508575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572309.508589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572309.520856:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86c0.
00000080:00000010:3.1:1788572309.520876:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526ad280.
00010000:00000010:0.1:1788572309.530687:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572309.530698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:0.1:1788572309.545020:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d00.
00000080:00000010:0.1:1788572309.545039:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526a8840.
00010000:00000010:3.1:1788572309.553428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572309.553448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00010000:00000010:0.1:1788572309.691001:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572309.691011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572309.694765:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb420.
00000080:00000010:0.1:1788572309.694792:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349757380.
00010000:00000010:1.1:1788572309.717473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.717490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:1.1:1788572309.717496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa50e0.
00000080:00000010:1.1:1788572309.717511:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497ac200.
00010000:00000010:1.1:1788572309.723596:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.723616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572309.746235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50e0.
00000080:00000010:1.1:1788572309.746263:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346211080.
00010000:00000010:1.1:1788572309.751482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.751498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572309.760667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00000080:00000010:1.1:1788572309.760694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346326b40.
00010000:00000010:1.1:1788572309.767652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572309.767669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:0.1:1788572309.992694:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572309.992710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:0.1:1788572310.014680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb680.
00000080:00000010:0.1:1788572310.019465:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b0840.
00010000:00000010:0.1:1788572310.024477:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572310.024496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572310.034067:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5780.
00000080:00000010:0.1:1788572310.034096:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519ddac0.
00010000:00000010:0.1:1788572310.063652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572310.063671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:1.1:1788572310.094449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5dc0.
00000080:00000010:1.1:1788572310.094480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354546b40.
00010000:00000010:3.1:1788572310.237569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab700.
00010000:00000010:3.1:1788572310.237595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572310.251799:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfc0.
00000080:00000010:3.1:1788572310.251827:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534600b9c0.
00010000:00000010:3.1:1788572310.362963:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:3.1:1788572310.362989:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572310.380438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf80.
00000080:00000010:3.1:1788572310.380469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535453a940.
00010000:00000010:3.1:1788572310.388700:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572310.388716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572310.402446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8760.
00000080:00000010:3.1:1788572310.402476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463bb180.
00010000:00000010:3.1:1788572310.419763:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572310.419772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572310.434295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572310.434324:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965dac0.
00010000:00000010:0.1:1788572310.449040:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572310.449117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:2.1:1788572310.488999:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8760.
00000080:00000010:2.1:1788572310.489019:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526ae300.
00010000:00000010:2.1:1788572310.494883:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.494891:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572310.502378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8e0.
00000080:00000010:2.1:1788572310.502426:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533c2940.
00010000:00000010:2.1:1788572310.521447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.521462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572310.534445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd60.
00000080:00000010:2.1:1788572310.538519:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b0a940.
00010000:00000010:2.1:1788572310.545436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.545451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572310.557794:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8e0.
00000080:00000010:2.1:1788572310.557819:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535180a100.
00010000:00000010:2.1:1788572310.570735:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.570750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572310.592000:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd60.
00000080:00000010:2.1:1788572310.592023:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a13180.
00010000:00000010:3.1:1788572310.601463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572310.601480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572310.616753:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8e0.
00000080:00000010:3.1:1788572310.616848:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351908840.
00010000:00000010:3.1:1788572310.624575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572310.624651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572310.655885:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572310.655903:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346399080.
00010000:00000010:2.1:1788572310.673486:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.673506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572310.681781:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8660.
00000080:00000010:2.1:1788572310.681805:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352688000.
00010000:00000010:2.1:1788572310.694475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.694491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572310.703979:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd60.
00000080:00000010:2.1:1788572310.704022:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be3180.
00010000:00000010:2.1:1788572310.717476:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.717494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572310.725950:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572310.725978:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423439c0.
00010000:00000010:2.1:1788572310.741440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.741457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572310.760747:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd60.
00000080:00000010:3.1:1788572310.760784:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544e6300.
00010000:00000010:3.1:1788572310.770450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:3.1:1788572310.770468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572310.781503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572310.781531:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349494a40.
00010000:00000010:0.1:1788572310.796651:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abe00.
00010000:00000010:0.1:1788572310.796666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:2.1:1788572310.806083:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8400.
00000080:00000010:2.1:1788572310.806113:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba0840.
00010000:00000010:2.1:1788572310.814687:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.814704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572310.824718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572310.828854:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349632940.
00010000:00000010:2.1:1788572310.845445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.845461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572310.852562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb300.
00000080:00000010:2.1:1788572310.852584:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495918c0.
00010000:00000010:2.1:1788572310.860447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.860463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572310.884569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572310.884596:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544fb180.
00010000:00000010:2.1:1788572310.895439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.895455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572310.917153:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb300.
00000080:00000010:2.1:1788572310.917174:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534234f380.
00010000:00000010:2.1:1788572310.921620:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.921637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572310.931523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572310.935836:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a818c0.
00010000:00000010:2.1:1788572310.940798:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.940806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:0.1:1788572310.957440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb300.
00000080:00000010:0.1:1788572310.962451:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534212d280.
00010000:00000010:2.1:1788572310.972346:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572310.972354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:2.1:1788572310.978559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a80.
00000080:00000010:2.1:1788572310.978589:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546f1080.
00010000:00000010:0.1:1788572310.982450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572310.982468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572311.001866:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572311.001895:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463da100.
00010000:00000010:2.1:1788572311.012446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.012455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572311.023445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba40.
00000080:00000010:2.1:1788572311.023472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d6300.
00010000:00000010:2.1:1788572311.027442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.027457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:2.1:1788572311.037504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572311.037552:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342086b40.
00010000:00000010:2.1:1788572311.048922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.048939:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:2.1:1788572311.058450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba40.
00000080:00000010:2.1:1788572311.062441:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534638a100.
00010000:00000010:2.1:1788572311.079961:0:1377:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.079979:0:1377:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:2.1:1788572311.098577:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572311.098601:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354761080.
00010000:00000010:3.1:1788572311.125261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572311.125268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572311.125273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba40.
00000080:00000010:3.1:1788572311.125287:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534944c200.
00010000:00000010:2.1:1788572311.134833:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abd00.
00010000:00000010:2.1:1788572311.134852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:2.1:1788572311.156696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:2.1:1788572311.156713:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534639eb40.
00010000:00000010:2.1:1788572311.160557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.160574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572311.170658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572311.175551:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519df380.
00010000:00000010:2.1:1788572311.201519:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.201528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572311.210768:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe00.
00000080:00000010:2.1:1788572311.210788:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349626300.
00010000:00000010:2.1:1788572311.214818:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.214835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572311.233626:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572311.233658:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346386300.
00010000:00000010:2.1:1788572311.233668:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.233674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572311.250848:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe00.
00000080:00000010:2.1:1788572311.250886:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354617380.
00010000:00000010:3.1:1788572311.265578:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572311.265593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572311.279430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:3.1:1788572311.279457:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349496300.
00010000:00000010:3.1:1788572311.300139:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572311.300153:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572311.303558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d89a0.
00000080:00000010:3.1:1788572311.303585:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346218000.
00010000:00000010:3.1:1788572311.321854:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572311.321870:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572311.351532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00000080:00000010:3.1:1788572311.351564:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421a0840.
00010000:00000010:2.1:1788572311.363559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.363577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:2.1:1788572311.376624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d89a0.
00000080:00000010:2.1:1788572311.376647:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342121080.
00010000:00000010:2.1:1788572311.393712:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.393723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572311.406840:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe00.
00000080:00000010:2.1:1788572311.408657:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546f4a40.
00010000:00000010:2.1:1788572311.422579:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.422595:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572311.429504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb000.
00000080:00000010:3.1:1788572311.429523:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544a5280.
00010000:00000010:3.1:1788572311.448441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572311.448457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572311.464244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00000080:00000010:3.1:1788572311.464272:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a42940.
00010000:00000010:3.1:1788572311.497434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572311.497503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572311.505491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572311.505515:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b33180.
00010000:00000010:2.1:1788572311.509754:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.509770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:2.1:1788572311.521305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00000080:00000010:2.1:1788572311.521325:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a9ca40.
00010000:00000010:2.1:1788572311.554243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572311.554264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572311.570727:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:2.1:1788572311.570743:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535455d280.
00010000:00000010:0.1:1788572311.577653:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:0.1:1788572311.577668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572311.586093:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb560.
00000080:00000010:0.1:1788572311.586119:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b08000.
00010000:00000010:1.1:1788572311.587748:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572311.587760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:1.1:1788572311.601544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53a0.
00000080:00000010:1.1:1788572311.601571:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965ca40.
00010000:00000010:2.1:1788572311.619376:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572311.619404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:2.1:1788572311.632693:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5520.
00000080:00000010:2.1:1788572311.632720:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ae3180.
00010000:00000010:2.1:1788572311.665451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572311.665460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572311.677505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:2.1:1788572311.677523:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aed280.
00010000:00000010:2.1:1788572311.681555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572311.681570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572311.697053:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:2.1:1788572311.697093:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353235280.
00010000:00000010:2.1:1788572311.713484:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572311.713500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572311.718452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:2.1:1788572311.718481:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534623d280.
00010000:00000010:2.1:1788572311.730673:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572311.730691:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:2.1:1788572311.746754:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:2.1:1788572311.754218:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463439c0.
00010000:00000010:2.1:1788572311.763471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572311.763495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572311.776443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:2.1:1788572311.776472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354506b40.
00010000:00000010:2.1:1788572311.790820:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572311.790836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572311.814816:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:2.1:1788572311.814840:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a518c0.
00010000:00000010:2.1:1788572311.824456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572311.824473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:0.1:1788572311.838451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00000080:00000010:0.1:1788572311.838482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544098c0.
00010000:00000010:3.1:1788572311.855454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572311.855472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:3.1:1788572311.866735:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa57a0.
00000080:00000010:3.1:1788572311.866759:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354501080.
00010000:00000010:3.1:1788572311.886483:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:3.1:1788572311.886497:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:2.1:1788572311.908804:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:2.1:1788572311.908830:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354766300.
00010000:00000010:2.1:1788572311.915610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572311.915636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572311.932498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:2.1:1788572311.932519:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463aa100.
00010000:00000010:0.1:1788572311.943790:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572311.943810:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572311.952432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00000080:00000010:2.1:1788572311.952583:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346316300.
00010000:00000010:2.1:1788572311.959897:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572311.959903:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572311.970429:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:2.1:1788572311.970462:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535192dac0.
00010000:00000010:2.1:1788572311.983656:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572311.983672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572311.986566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00000080:00000010:2.1:1788572311.986595:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b75ac0.
00010000:00000010:2.1:1788572311.986604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572311.986608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572311.994431:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:2.1:1788572311.994459:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a18c0.
00010000:00000010:2.1:1788572312.004498:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572312.004505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572312.013723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00000080:00000010:2.1:1788572312.013754:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535326ca40.
00010000:00000010:0.1:1788572312.028427:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:0.1:1788572312.028435:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572312.037463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:2.1:1788572312.037499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535475a100.
00010000:00000010:2.1:1788572312.077557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572312.077576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:1.1:1788572312.083985:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00000080:00000010:1.1:1788572312.084015:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535441f380.
00010000:00000010:1.1:1788572312.106717:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572312.106731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572312.123703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5fe0.
00000080:00000010:1.1:1788572312.123729:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535197a940.
00010000:00000010:1.1:1788572312.133558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572312.133576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572312.146914:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d40.
00000080:00000010:1.1:1788572312.146939:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535447b9c0.
00010000:00000010:1.1:1788572312.164640:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572312.164655:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572312.176430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5fe0.
00000080:00000010:1.1:1788572312.182172:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb2940.
00010000:00000010:1.1:1788572312.185811:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572312.185828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:1.1:1788572312.212433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d40.
00000080:00000010:1.1:1788572312.212704:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535275a940.
00010000:00000010:0.1:1788572312.237799:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab200.
00010000:00000010:0.1:1788572312.237879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:2.1:1788572312.242812:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5fe0.
00000080:00000010:2.1:1788572312.260510:0:13:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351998000.
00000080:00000010:1.1:1788572312.268940:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb2a0.
00000080:00000010:1.1:1788572312.268968:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535275b180.
00010000:00000010:0.1:1788572312.269213:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572312.269221:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:2.1:1788572312.273589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572312.273605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:2.1:1788572312.294428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54c0.
00000080:00000010:2.1:1788572312.307533:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342236b40.
00010000:00000010:2.1:1788572312.307547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572312.307553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572312.311515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3a0.
00000080:00000010:2.1:1788572312.311543:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544898c0.
00010000:00000010:2.1:1788572312.311554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572312.311559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572312.327432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1c0.
00000080:00000010:2.1:1788572312.327450:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349740000.
00010000:00000010:1.1:1788572312.348431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572312.348439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:1.1:1788572312.352453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3a0.
00000080:00000010:1.1:1788572312.352480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534632f380.
00010000:00000010:1.1:1788572312.363722:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:1.1:1788572312.363739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572312.394265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5740.
00000080:00000010:1.1:1788572312.394291:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b56300.
00010000:00000010:2.1:1788572312.411449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074300.
00010000:00000010:2.1:1788572312.411465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:2.1:1788572312.428575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58c0.
00000080:00000010:2.1:1788572312.428611:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421ab180.
00010000:00000010:2.1:1788572312.452487:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572312.452500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572312.496644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1c0.
00000080:00000010:3.1:1788572312.496670:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352695ac0.
00010000:00000010:3.1:1788572312.520634:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572312.520659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572312.530787:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8700.
00000080:00000010:3.1:1788572312.530816:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351978000.
00010000:00000010:3.1:1788572312.530827:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572312.530833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572312.551921:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c00.
00000080:00000010:3.1:1788572312.557892:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba1080.
00010000:00000010:2.1:1788572312.568840:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572312.568857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:2.1:1788572312.591593:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8700.
00000080:00000010:2.1:1788572312.591620:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bfca40.
00010000:00000010:2.1:1788572312.602433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572312.602449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572312.617288:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4a0.
00000080:00000010:3.1:1788572312.617315:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535446d280.
00010000:00000010:3.1:1788572312.634387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572312.634417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:3.1:1788572312.646556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c00.
00000080:00000010:3.1:1788572312.646578:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351988840.
00010000:00000010:2.1:1788572312.658960:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572312.658978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:2.1:1788572312.682684:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:2.1:1788572312.682708:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421d7380.
00010000:00000010:2.1:1788572312.704454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572312.704465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572312.750055:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb160.
00000080:00000010:2.1:1788572312.750084:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d39c0.
00010000:00000010:2.1:1788572312.773884:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572312.773896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572312.777554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfa0.
00000080:00000010:2.1:1788572312.777592:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420818c0.
00010000:00000010:0.1:1788572312.792472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572312.792488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:0.1:1788572312.803258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb160.
00000080:00000010:0.1:1788572312.803288:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535199a940.
00010000:00000010:3.1:1788572312.824677:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572312.824692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:3.1:1788572312.856676:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5bc0.
00000080:00000010:3.1:1788572312.856705:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351971080.
00010000:00000010:3.1:1788572312.869671:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572312.869687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572312.879439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c00.
00000080:00000010:3.1:1788572312.879465:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349756300.
00010000:00000010:2.1:1788572312.890620:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572312.890635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:2.1:1788572312.933590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8720.
00000080:00000010:2.1:1788572312.933620:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346338000.
00010000:00000010:1.1:1788572312.950617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572312.950633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:1.1:1788572312.966435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfa0.
00000080:00000010:1.1:1788572312.966459:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b4ca40.
00010000:00000010:2.1:1788572312.974448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572312.974462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:2.1:1788572312.981677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57e0.
00000080:00000010:2.1:1788572312.981706:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342184a40.
00010000:00000010:3.1:1788572313.008330:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:3.1:1788572313.008344:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572313.049437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbaa0.
00000080:00000010:2.1:1788572313.057548:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421839c0.
00010000:00000010:2.1:1788572313.070029:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572313.070045:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572313.087745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb180.
00000080:00000010:2.1:1788572313.087772:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a6f380.
00010000:00000010:2.1:1788572313.103458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572313.103476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572313.129469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:2.1:1788572313.129499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534602d280.
00010000:00000010:3.1:1788572313.150323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:3.1:1788572313.150337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572313.159570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb180.
00000080:00000010:3.1:1788572313.159612:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620b9c0.
00010000:00000010:2.1:1788572313.186488:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:2.1:1788572313.186503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:2.1:1788572313.194779:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c00.
00000080:00000010:2.1:1788572313.194813:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354764a40.
00010000:00000010:2.1:1788572313.208545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572313.208560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572313.232469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:2.1:1788572313.232495:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535330f380.
00010000:00000010:2.1:1788572313.253443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572313.253457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572313.294464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb960.
00000080:00000010:3.1:1788572313.294488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534952e300.
00010000:00000010:2.1:1788572313.313436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572313.313447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:2.1:1788572313.348600:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c00.
00000080:00000010:2.1:1788572313.348629:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519eeb40.
00010000:00000010:2.1:1788572313.366253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572313.366268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572313.371021:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:2.1:1788572313.371048:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353290000.
00010000:00000010:2.1:1788572313.388601:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572313.388610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572313.405488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf40.
00000080:00000010:2.1:1788572313.405516:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535329c200.
00010000:00000010:0.1:1788572313.419710:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:0.1:1788572313.419729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:0.1:1788572313.423832:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:0.1:1788572313.423851:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b70840.
00010000:00000010:1.1:1788572313.436782:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572313.436790:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:1.1:1788572313.444449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c00.
00000080:00000010:1.1:1788572313.444476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532439c0.
00010000:00000010:0.1:1788572313.453617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572313.453633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:0.1:1788572313.460779:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c52a0.
00000080:00000010:0.1:1788572313.460809:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ab6300.
00000080:00000010:3.1:1788572313.476461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55c0.
00000080:00000010:3.1:1788572313.476498:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351abc200.
00010000:00000010:0.1:1788572313.476750:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572313.476759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00010000:00000010:1.1:1788572313.495685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572313.495698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:1.1:1788572313.507510:0:206940:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8720.
00000080:00000010:1.1:1788572313.507525:0:206940:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346200000.
00010000:00000010:1.1:1788572313.516461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572313.516478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:2.1:1788572313.516580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58e0.
00000080:00000010:2.1:1788572313.516601:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be0000.
00010000:00000010:2.1:1788572313.532506:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572313.532523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572313.537586:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf40.
00000080:00000010:2.1:1788572313.537605:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544b5ac0.
00010000:00000010:2.1:1788572313.541569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572313.541585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572313.568415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:2.1:1788572313.568434:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bd6300.
00010000:00000010:1.1:1788572313.573626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572313.573639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:1.1:1788572313.581841:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf40.
00000080:00000010:1.1:1788572313.581869:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352512940.
00010000:00000010:1.1:1788572313.584099:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572313.584106:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:1.1:1788572313.588218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5800.
00000080:00000010:1.1:1788572313.588249:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349623180.
00000080:00000010:1.1:1788572313.611613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53c0.
00000080:00000010:1.1:1788572313.611630:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533e5280.
00010000:00000010:0.1:1788572313.611822:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572313.611829:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00010000:00000010:0.1:1788572313.617464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572313.617479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:0.1:1788572313.625563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ca0.
00000080:00000010:0.1:1788572313.625590:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f8000.
00010000:00000010:0.1:1788572313.637655:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572313.637679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572313.656559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5d00.
00000080:00000010:0.1:1788572313.656577:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423618c0.
00010000:00000010:0.1:1788572313.660626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572313.660644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572313.676460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a00.
00000080:00000010:0.1:1788572313.676491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342363180.
00010000:00000010:0.1:1788572313.688449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572313.688465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:1.1:1788572313.703443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5d00.
00000080:00000010:1.1:1788572313.703471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544bc200.
00010000:00000010:2.1:1788572313.713496:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572313.713513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:2.1:1788572313.728416:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58e0.
00000080:00000010:2.1:1788572313.732667:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544edac0.
00010000:00000010:2.1:1788572313.737739:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572313.737755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572313.753553:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:2.1:1788572313.753570:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495af380.
00010000:00000010:2.1:1788572313.758589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572313.758602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572313.794488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572313.794518:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b7ca40.
00010000:00000010:0.1:1788572313.794582:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:0.1:1788572313.794589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:1.1:1788572313.803454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:1.1:1788572313.803488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e3180.
00010000:00000010:1.1:1788572313.815445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572313.815461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572313.858852:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5de0.
00000080:00000010:1.1:1788572313.858878:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346041080.
00010000:00000010:1.1:1788572313.865848:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572313.865864:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572313.880419:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5e20.
00000080:00000010:1.1:1788572313.880436:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534604d280.
00010000:00000010:1.1:1788572313.886724:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572313.886738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572313.900466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5de0.
00000080:00000010:1.1:1788572313.900494:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351921080.
00010000:00000010:1.1:1788572313.920602:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572313.920610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:1.1:1788572313.932092:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5e20.
00000080:00000010:1.1:1788572313.932116:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544a39c0.
00010000:00000010:0.1:1788572313.947157:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572313.947174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:0.1:1788572313.960052:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5de0.
00000080:00000010:0.1:1788572313.960092:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535337a940.
00010000:00000010:0.1:1788572313.960102:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572313.960108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572313.964082:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5020.
00000080:00000010:0.1:1788572313.964118:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b10000.
00010000:00000010:1.1:1788572313.968745:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:1.1:1788572313.968765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:2.1:1788572313.978476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5fe0.
00000080:00000010:2.1:1788572313.978506:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534222b9c0.
00010000:00000010:2.1:1788572313.986566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572313.986582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572314.011171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572314.011198:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494eca40.
00010000:00000010:2.1:1788572314.015876:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.015891:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572314.034518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6a0.
00000080:00000010:2.1:1788572314.068694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533f6b40.
00010000:00000010:2.1:1788572314.068704:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.068709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572314.069690:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572314.069709:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354539080.
00010000:00000010:3.1:1788572314.073592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572314.073608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:3.1:1788572314.107120:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6a0.
00000080:00000010:3.1:1788572314.107149:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a27380.
00010000:00000010:2.1:1788572314.119812:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.119829:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:2.1:1788572314.130522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d84c0.
00000080:00000010:2.1:1788572314.135794:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a53180.
00010000:00000010:2.1:1788572314.149436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.149458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572314.164468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572314.172765:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aca940.
00010000:00000010:0.1:1788572314.196630:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:0.1:1788572314.196643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572314.212738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:2.1:1788572314.212770:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e6300.
00010000:00000010:2.1:1788572314.218546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.218562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572314.245456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572314.245488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b718c0.
00010000:00000010:2.1:1788572314.259487:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.259501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572314.278444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:2.1:1788572314.282483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497ad280.
00010000:00000010:3.1:1788572314.302869:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572314.302884:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572314.334604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:3.1:1788572314.338705:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497a3180.
00010000:00000010:2.1:1788572314.349513:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.349530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:2.1:1788572314.371872:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8700.
00000080:00000010:2.1:1788572314.376671:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b18840.
00010000:00000010:2.1:1788572314.398545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.398559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572314.402537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:2.1:1788572314.402565:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534602eb40.
00010000:00000010:2.1:1788572314.402575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572314.402581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572314.417538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:2.1:1788572314.417563:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533c5ac0.
00010000:00000010:2.1:1788572314.438436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572314.438464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572314.447660:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:2.1:1788572314.450982:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b2f380.
00010000:00000010:2.1:1788572314.484887:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572314.484902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572314.496035:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:2.1:1788572314.514491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a49080.
00010000:00000010:3.1:1788572314.519446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572314.519465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572314.544429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:2.1:1788572314.544593:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e1080.
00010000:00000010:3.1:1788572314.550450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572314.550459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572314.567520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:2.1:1788572314.581591:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953525898c0.
00010000:00000010:1.1:1788572314.585554:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:1.1:1788572314.585562:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:1.1:1788572314.595524:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:1.1:1788572314.595548:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533f1080.
00010000:00000010:1.1:1788572314.604204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572314.604221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:2.1:1788572314.621478:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ae0.
00000080:00000010:2.1:1788572314.621497:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a5a940.
00010000:00000010:2.1:1788572314.627457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.627474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572314.653969:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:2.1:1788572314.653998:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e3180.
00010000:00000010:2.1:1788572314.661464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.661472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572314.665457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:2.1:1788572314.665585:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532498c0.
00010000:00000010:3.1:1788572314.683297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572314.683312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572314.704453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:3.1:1788572314.704487:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353243180.
00010000:00000010:3.1:1788572314.704496:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572314.704502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572314.736060:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572314.736076:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aba100.
00010000:00000010:3.1:1788572314.736089:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572314.736103:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:1.1:1788572314.750802:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534639b9c0.
00000080:00000010:0.1:1788572314.753797:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86a0.
00010000:00000010:0.1:1788572314.759300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572314.759315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:0.1:1788572314.767651:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c20.
00000080:00000010:0.1:1788572314.767677:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bd5280.
00010000:00000010:0.1:1788572314.781452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572314.781468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572314.795458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5380.
00000080:00000010:0.1:1788572314.798779:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349440840.
00010000:00000010:0.1:1788572314.808387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572314.808414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572314.826485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b60.
00000080:00000010:0.1:1788572314.826507:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953525139c0.
00010000:00000010:1.1:1788572314.831075:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:1.1:1788572314.831087:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:1.1:1788572314.840476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5380.
00000080:00000010:1.1:1788572314.840510:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ea100.
00010000:00000010:0.1:1788572314.860547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:0.1:1788572314.860561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:2.1:1788572314.878436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5860.
00000080:00000010:2.1:1788572314.878461:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352516300.
00010000:00000010:2.1:1788572314.890442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.890459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572314.902471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:2.1:1788572314.902499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422db9c0.
00010000:00000010:2.1:1788572314.917531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.917547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572314.932180:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbe0.
00000080:00000010:2.1:1788572314.932204:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d4200.
00010000:00000010:2.1:1788572314.941439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572314.941467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572314.970430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:2.1:1788572314.970449:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535443b9c0.
00010000:00000010:3.1:1788572314.976566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572314.976583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572314.995445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbe0.
00000080:00000010:3.1:1788572314.999588:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a8840.
00010000:00000010:3.1:1788572315.003439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572315.003455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572315.015467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572315.015496:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519a5ac0.
00010000:00000010:3.1:1788572315.051240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572315.051255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:2.1:1788572315.062425:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494fa940.
00000080:00000010:3.1:1788572315.066123:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d89a0.
00010000:00000010:2.1:1788572315.071491:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572315.071507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572315.083559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:2.1:1788572315.083584:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353228840.
00010000:00000010:2.1:1788572315.090542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572315.090555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572315.113408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:2.1:1788572315.118026:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ab9080.
00010000:00000010:2.1:1788572315.122458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572315.122475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572315.138202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:2.1:1788572315.138229:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ab0840.
00010000:00000010:3.1:1788572315.148443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:3.1:1788572315.148461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572315.168520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:3.1:1788572315.177624:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535324a940.
00010000:00000010:3.1:1788572315.185000:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572315.185016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:2.1:1788572315.203304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d89a0.
00000080:00000010:2.1:1788572315.203334:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544eeb40.
00010000:00000010:2.1:1788572315.208829:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572315.208840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572315.213702:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:2.1:1788572315.213738:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495aca40.
00010000:00000010:2.1:1788572315.241819:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572315.241839:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572315.254492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:2.1:1788572315.254517:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535443ca40.
00010000:00000010:3.1:1788572315.275899:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572315.275914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572315.281704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:3.1:1788572315.281747:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544b4200.
00010000:00000010:3.1:1788572315.294513:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572315.294528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:2.1:1788572315.301574:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349664200.
00000080:00000010:3.1:1788572315.306448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00010000:00000010:2.1:1788572315.325594:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572315.325610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572315.367621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:2.1:1788572315.367654:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342367380.
00010000:00000010:2.1:1788572315.372454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572315.372469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572315.376467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:2.1:1788572315.376497:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533e2100.
00010000:00000010:2.1:1788572315.376504:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572315.376509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572315.395014:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572315.395042:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544ff380.
00010000:00000010:2.1:1788572315.404750:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572315.404765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572315.419634:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:2.1:1788572315.419665:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349441080.
00010000:00000010:3.1:1788572315.433653:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572315.433667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572315.440546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:3.1:1788572315.440577:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bd0000.
00010000:00000010:3.1:1788572315.449459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572315.449476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572315.481453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572315.487444:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be6300.
00010000:00000010:2.1:1788572315.492590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572315.492605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572315.514807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8740.
00000080:00000010:2.1:1788572315.514829:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534214d280.
00010000:00000010:0.1:1788572315.537706:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572315.537836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:0.1:1788572315.549241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:0.1:1788572315.549259:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346207380.
00010000:00000010:0.1:1788572315.549288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572315.549293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572315.553558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5640.
00000080:00000010:0.1:1788572315.553592:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351922100.
00010000:00000010:0.1:1788572315.566589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572315.566600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:2.1:1788572315.593630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa59c0.
00000080:00000010:2.1:1788572315.593658:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534604dac0.
00010000:00000010:2.1:1788572315.602292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572315.602307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572315.604626:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572315.604653:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346048840.
00010000:00000010:2.1:1788572315.608448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572315.608468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572315.620202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:2.1:1788572315.627163:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532e3180.
00010000:00000010:2.1:1788572315.636580:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572315.636590:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572315.643351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572315.643384:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351acb180.
00010000:00000010:3.1:1788572315.658028:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:3.1:1788572315.658044:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572315.670583:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:2.1:1788572315.674591:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a54200.
00010000:00000010:2.1:1788572315.695278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572315.695294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572315.719501:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572315.719526:0:13:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a218c0.
00010000:00000010:3.1:1788572315.727698:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572315.727710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:0.1:1788572315.731573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:0.1:1788572315.731597:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533f0840.
00010000:00000010:1.1:1788572315.735615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572315.735633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:1.1:1788572315.749530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa52a0.
00000080:00000010:1.1:1788572315.749561:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ad280.
00010000:00000010:1.1:1788572315.765279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:1.1:1788572315.765290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:1.1:1788572315.768838:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ac0.
00000080:00000010:1.1:1788572315.768872:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494e8000.
00010000:00000010:0.1:1788572315.782454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:0.1:1788572315.782470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:2.1:1788572315.794436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54a0.
00000080:00000010:2.1:1788572315.794477:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b17380.
00010000:00000010:2.1:1788572315.804452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572315.804468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572315.814251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572315.814279:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534222dac0.
00010000:00000010:3.1:1788572315.831706:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:3.1:1788572315.831713:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572315.837597:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8a0.
00000080:00000010:3.1:1788572315.837616:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535337eb40.
00010000:00000010:3.1:1788572315.844429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572315.844679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572315.852640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:3.1:1788572315.852669:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421bca40.
00010000:00000010:2.1:1788572315.865051:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:2.1:1788572315.865066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:2.1:1788572315.876558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8400.
00000080:00000010:2.1:1788572315.876586:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b22940.
00010000:00000010:2.1:1788572315.894797:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572315.894806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572315.903108:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572315.903144:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a25ac0.
00010000:00000010:2.1:1788572315.923882:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572315.923894:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:2.1:1788572315.932896:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbec0.
00000080:00000010:2.1:1788572315.932929:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534604ca40.
00010000:00000010:2.1:1788572315.932950:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572315.932955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572315.932959:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572315.932988:0:2353:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497a8000.
00010000:00000010:3.1:1788572315.935634:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572315.935648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:3.1:1788572315.957104:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb880.
00000080:00000010:3.1:1788572315.957144:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b7d280.
00010000:00000010:0.1:1788572315.969545:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:0.1:1788572315.969563:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:0.1:1788572315.982625:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8620.
00000080:00000010:0.1:1788572315.982660:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ac98c0.
00010000:00000010:0.1:1788572315.990068:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572315.990082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:1.1:1788572316.003608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5320.
00000080:00000010:1.1:1788572316.003629:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534222ca40.
00010000:00000010:2.1:1788572316.007428:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572316.007435:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:2.1:1788572316.007440:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5460.
00000080:00000010:2.1:1788572316.007465:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a23180.
00010000:00000010:2.1:1788572316.012758:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572316.012772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572316.025107:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572316.027583:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535258d280.
00010000:00000010:2.1:1788572316.040955:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572316.040968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572316.044473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbec0.
00000080:00000010:2.1:1788572316.044490:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a46300.
00010000:00000010:2.1:1788572316.053756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572316.053775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572316.067431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572316.067458:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e5ac0.
00010000:00000010:0.1:1788572316.083407:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:0.1:1788572316.083424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:0.1:1788572316.087570:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbec0.
00000080:00000010:0.1:1788572316.087587:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533c1080.
00010000:00000010:0.1:1788572316.096077:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572316.096094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572316.116569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5400.
00000080:00000010:0.1:1788572316.116596:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b2e300.
00010000:00000010:0.1:1788572316.134569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:0.1:1788572316.134584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572316.155640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b00.
00000080:00000010:0.1:1788572316.155659:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e6b40.
00010000:00000010:2.1:1788572316.164569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572316.164586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:3.1:1788572316.202567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5400.
00000080:00000010:3.1:1788572316.208649:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b1eb40.
00010000:00000010:1.1:1788572316.223442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:1.1:1788572316.223460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:1.1:1788572316.247443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8660.
00000080:00000010:1.1:1788572316.247474:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ab5ac0.
00010000:00000010:1.1:1788572316.257731:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572316.257758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:1.1:1788572316.274684:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5740.
00000080:00000010:1.1:1788572316.274716:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353240000.
00010000:00000010:2.1:1788572316.298620:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572316.298629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:2.1:1788572316.304670:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c58c0.
00000080:00000010:2.1:1788572316.304687:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535324b9c0.
00010000:00000010:2.1:1788572316.313662:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572316.313676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572316.331521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572316.331559:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bd6b40.
00010000:00000010:2.1:1788572316.335731:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572316.335745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572316.343475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:2.1:1788572316.343506:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533e6b40.
00010000:00000010:3.1:1788572316.362534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572316.362550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572316.384125:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:3.1:1788572316.384160:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519a39c0.
00010000:00000010:2.1:1788572316.392444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572316.392459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572316.420230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8760.
00000080:00000010:2.1:1788572316.420257:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535443dac0.
00010000:00000010:2.1:1788572316.435019:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572316.435036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572316.452451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:2.1:1788572316.452490:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342365ac0.
00010000:00000010:0.1:1788572316.462485:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:0.1:1788572316.462497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:1.1:1788572316.478205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb920.
00000080:00000010:1.1:1788572316.478230:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422da100.
00010000:00000010:0.1:1788572316.483009:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572316.483024:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:0.1:1788572316.490454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5080.
00000080:00000010:0.1:1788572316.490481:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be5280.
00010000:00000010:0.1:1788572316.508470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:0.1:1788572316.508479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:2.1:1788572316.512632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5640.
00000080:00000010:2.1:1788572316.512663:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d18c0.
00010000:00000010:2.1:1788572316.523440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572316.523457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572316.550572:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:2.1:1788572316.550591:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352514200.
00010000:00000010:3.1:1788572316.561374:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572316.561390:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572316.561422:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:3.1:1788572316.561437:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352702100.
00010000:00000010:3.1:1788572316.574427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572316.574434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:2.1:1788572316.585468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d86c0.
00000080:00000010:2.1:1788572316.585496:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352512100.
00010000:00000010:2.1:1788572316.599597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572316.599604:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572316.608835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:2.1:1788572316.608854:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346395ac0.
00010000:00000010:2.1:1788572316.608860:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572316.608864:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572316.612459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:2.1:1788572316.612484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349446b40.
00010000:00000010:2.1:1788572316.622454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572316.622471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572316.630968:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba80.
00000080:00000010:2.1:1788572316.640459:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354735ac0.
00010000:00000010:2.1:1788572316.647453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572316.647466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:0.1:1788572316.650607:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:0.1:1788572316.650632:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a57380.
00010000:00000010:1.1:1788572316.676444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:1.1:1788572316.676459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:2.1:1788572316.687455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa50a0.
00000080:00000010:2.1:1788572316.687479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535440e300.
00010000:00000010:2.1:1788572316.715432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572316.715438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:1.1:1788572316.724300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba80.
00000080:00000010:1.1:1788572316.724329:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354502940.
00010000:00000010:1.1:1788572316.734304:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572316.734321:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:0.1:1788572316.736638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5fc0.
00000080:00000010:0.1:1788572316.736661:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535440b180.
00010000:00000010:2.1:1788572316.742580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572316.742594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:2.1:1788572316.757461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a20.
00000080:00000010:2.1:1788572316.757490:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346345ac0.
00010000:00000010:2.1:1788572316.771794:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572316.771810:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572316.781459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:2.1:1788572316.789455:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342159080.
00010000:00000010:2.1:1788572316.794542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572316.794557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:0.1:1788572316.801459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0c0.
00000080:00000010:0.1:1788572316.801487:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b99080.
00010000:00000010:0.1:1788572316.821635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572316.821646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572316.839251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b80.
00010000:00000010:2.1:1788572316.844437:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00000080:00000010:0.1:1788572316.844450:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346312100.
00010000:00000010:2.1:1788572316.844597:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:2.1:1788572316.862317:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5000.
00000080:00000010:2.1:1788572316.862341:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463a9080.
00010000:00000010:2.1:1788572316.867618:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572316.867632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572316.873430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:2.1:1788572316.873455:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d4a40.
00010000:00000010:0.1:1788572316.883568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572316.883587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:0.1:1788572316.905559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:0.1:1788572316.905586:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535275a100.
00010000:00000010:0.1:1788572316.932091:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572316.932107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572316.947748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5860.
00000080:00000010:2.1:1788572316.952314:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351988000.
00010000:00000010:2.1:1788572316.960448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572316.960473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572316.982594:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:2.1:1788572316.987595:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527539c0.
00010000:00000010:2.1:1788572316.993453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572316.993466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572316.993473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572316.993490:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354478840.
00010000:00000010:2.1:1788572316.997447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572316.997524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572317.025598:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:2.1:1788572317.025640:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb6b40.
00010000:00000010:2.1:1788572317.038920:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.038949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572317.056453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572317.056483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535197eb40.
00010000:00000010:3.1:1788572317.071882:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572317.071900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:3.1:1788572317.093926:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:3.1:1788572317.097092:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535475a940.
00010000:00000010:3.1:1788572317.108588:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572317.108599:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:3.1:1788572317.125710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d00.
00000080:00000010:3.1:1788572317.125738:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535475b180.
00010000:00000010:2.1:1788572317.135613:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.135629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:2.1:1788572317.158903:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f60.
00000080:00000010:2.1:1788572317.158927:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a5280.
00010000:00000010:2.1:1788572317.168658:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.168673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572317.213481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572317.213545:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353268000.
00010000:00000010:2.1:1788572317.257672:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.257688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572317.297628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb060.
00000080:00000010:2.1:1788572317.297658:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421adac0.
00010000:00000010:2.1:1788572317.322811:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.322826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:3.1:1788572317.340138:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:3.1:1788572317.340174:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b50840.
00010000:00000010:3.1:1788572317.347525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572317.347542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572317.358436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d00.
00000080:00000010:3.1:1788572317.358470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534632dac0.
00010000:00000010:3.1:1788572317.386574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572317.386593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:2.1:1788572317.412447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d80.
00000080:00000010:2.1:1788572317.412486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534223b9c0.
00010000:00000010:2.1:1788572317.429338:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.429349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572317.435459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb060.
00000080:00000010:2.1:1788572317.435486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342230000.
00010000:00000010:2.1:1788572317.443306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.443323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572317.454658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572317.454683:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535199eb40.
00010000:00000010:2.1:1788572317.465611:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.465628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572317.493442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb060.
00000080:00000010:2.1:1788572317.493469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351972940.
00010000:00000010:2.1:1788572317.503570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.503586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:0.1:1788572317.516499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534217ca40.
00000080:00000010:2.1:1788572317.521447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00010000:00000010:0.1:1788572317.528698:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572317.528711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572317.539597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa51a0.
00000080:00000010:0.1:1788572317.539624:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d5ac0.
00010000:00000010:0.1:1788572317.565564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:0.1:1788572317.565581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572317.584892:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f20.
00000080:00000010:0.1:1788572317.584930:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421d0000.
00010000:00000010:2.1:1788572317.608454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.608471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:2.1:1788572317.635814:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5760.
00000080:00000010:2.1:1788572317.635841:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535327d280.
00010000:00000010:2.1:1788572317.651609:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572317.651624:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572317.671149:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572317.671202:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535198eb40.
00010000:00000010:0.1:1788572317.675519:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:0.1:1788572317.675533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:0.1:1788572317.680910:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb060.
00000080:00000010:0.1:1788572317.681103:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf8000.
00010000:00000010:2.1:1788572317.704829:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4000.
00010000:00000010:2.1:1788572317.704843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:2.1:1788572317.709643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5100.
00000080:00000010:2.1:1788572317.709670:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533a8840.
00010000:00000010:2.1:1788572317.727147:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572317.727158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572317.735713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:3.1:1788572317.735741:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352696b40.
00010000:00000010:3.1:1788572317.753824:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572317.753841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572317.757701:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d00.
00000080:00000010:3.1:1788572317.757729:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533d9080.
00010000:00000010:3.1:1788572317.780136:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572317.780147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:2.1:1788572317.785421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8580.
00000080:00000010:2.1:1788572317.788314:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349743180.
00010000:00000010:2.1:1788572317.803296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572317.803313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572317.807686:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:2.1:1788572317.807703:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342184200.
00010000:00000010:2.1:1788572317.818724:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572317.818731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:2.1:1788572317.830683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572317.830704:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b4c200.
00010000:00000010:2.1:1788572317.835450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:2.1:1788572317.835466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572317.865777:0:1377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:2.1:1788572317.865810:0:1377:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534633b9c0.
00010000:00000010:0.1:1788572317.872346:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572317.872362:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:1.1:1788572317.882708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:1.1:1788572317.882725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:0.1:1788572317.882954:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:0.1:1788572317.882981:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349752940.
00000080:00000010:2.1:1788572317.885774:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ba0.
00000080:00000010:2.1:1788572317.885894:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420ba100.
00010000:00000010:2.1:1788572317.890457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572317.890475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572317.899597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:2.1:1788572317.899620:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494b18c0.
00010000:00000010:2.1:1788572317.925559:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572317.925577:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572317.934625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572317.938646:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495418c0.
00010000:00000010:2.1:1788572317.945044:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572317.945060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572317.962640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:2.1:1788572317.962669:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354762940.
00010000:00000010:2.1:1788572317.982011:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572317.982026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572318.006991:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572318.007020:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3f380.
00010000:00000010:0.1:1788572318.032573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:0.1:1788572318.032587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:0.1:1788572318.036594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:0.1:1788572318.053955:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534602c200.
00010000:00000010:0.1:1788572318.070664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572318.070673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572318.070679:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e40.
00000080:00000010:0.1:1788572318.079320:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346344a40.
00010000:00000010:0.1:1788572318.084214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572318.084232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572318.099595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5aa0.
00000080:00000010:0.1:1788572318.099633:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354500000.
00010000:00000010:2.1:1788572318.110609:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.110625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.134450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e40.
00000080:00000010:2.1:1788572318.134474:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b76300.
00010000:00000010:2.1:1788572318.172214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.172231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.175568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572318.175596:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535269c200.
00010000:00000010:2.1:1788572318.197529:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.197538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572318.210463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:2.1:1788572318.210491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463a1080.
00010000:00000010:2.1:1788572318.226954:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.226971:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572318.241636:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572318.241679:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353292100.
00010000:00000010:2.1:1788572318.271931:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.271958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572318.271970:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:2.1:1788572318.271988:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349560000.
00010000:00000010:2.1:1788572318.304492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.304510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.312596:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572318.312616:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535455f380.
00010000:00000010:2.1:1788572318.325788:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.325813:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572318.342124:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:2.1:1788572318.342149:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b0b180.
00010000:00000010:2.1:1788572318.359436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.359457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.359464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572318.359479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474a940.
00010000:00000010:2.1:1788572318.364471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.364488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572318.364493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:2.1:1788572318.364511:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965d280.
00010000:00000010:2.1:1788572318.372825:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.372840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572318.381458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572318.381492:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953545398c0.
00010000:00000010:2.1:1788572318.393614:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.393629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572318.404545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:2.1:1788572318.404571:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533f39c0.
00010000:00000010:2.1:1788572318.419718:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.419735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572318.431701:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572318.431719:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535180a940.
00010000:00000010:2.1:1788572318.442722:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.442730:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572318.451586:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:2.1:1788572318.451606:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534605b9c0.
00010000:00000010:2.1:1788572318.456448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.456466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572318.468462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572318.468486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a43180.
00010000:00000010:3.1:1788572318.480919:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572318.480938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572318.493457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:2.1:1788572318.493483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535190a100.
00010000:00000010:3.1:1788572318.506415:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572318.506422:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572318.515949:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:3.1:1788572318.515982:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a12100.
00010000:00000010:2.1:1788572318.526699:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.526708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00000080:00000010:2.1:1788572318.530655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d00.
00000080:00000010:2.1:1788572318.530684:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346380840.
00010000:00000010:2.1:1788572318.546665:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.546682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.555463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:2.1:1788572318.555489:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534234dac0.
00010000:00000010:2.1:1788572318.563559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.563577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.574115:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572318.574143:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349590840.
00010000:00000010:2.1:1788572318.591786:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572318.591805:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.605524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4c0.
00000080:00000010:2.1:1788572318.605552:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349495ac0.
00010000:00000010:0.1:1788572318.611932:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:0.1:1788572318.611946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:0.1:1788572318.614436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:0.1:1788572318.614469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463c7380.
00010000:00000010:3.1:1788572318.629991:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572318.630006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:2.1:1788572318.637719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c00.
00000080:00000010:2.1:1788572318.637752:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349490840.
00010000:00000010:2.1:1788572318.649657:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.649675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.654431:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf20.
00000080:00000010:2.1:1788572318.654448:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342344a40.
00010000:00000010:2.1:1788572318.659558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.659576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572318.671076:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572318.671094:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544e0840.
00010000:00000010:1.1:1788572318.685701:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572318.685717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:1.1:1788572318.685723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c00.
00000080:00000010:1.1:1788572318.685741:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351be7380.
00010000:00000010:1.1:1788572318.699543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572318.699561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:2.1:1788572318.720931:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5200.
00000080:00000010:2.1:1788572318.723644:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463df380.
00010000:00000010:2.1:1788572318.728540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572318.728557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572318.737458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572318.737483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519deb40.
00010000:00000010:0.1:1788572318.748475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:0.1:1788572318.748491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:0.1:1788572318.758654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf20.
00000080:00000010:0.1:1788572318.758693:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544ce300.
00010000:00000010:2.1:1788572318.775525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.775534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:2.1:1788572318.779955:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ee0.
00000080:00000010:2.1:1788572318.779989:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546f2940.
00010000:00000010:2.1:1788572318.788652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.788668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.795666:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572318.795686:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421298c0.
00010000:00000010:2.1:1788572318.817550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.817557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.829719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:2.1:1788572318.829735:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a898c0.
00010000:00000010:0.1:1788572318.846836:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572318.846847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572318.846853:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:0.1:1788572318.846867:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421a1080.
00010000:00000010:0.1:1788572318.850694:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572318.850705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:2.1:1788572318.851415:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b40.
00000080:00000010:2.1:1788572318.851457:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544eca40.
00010000:00000010:2.1:1788572318.870238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.870257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.876435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:2.1:1788572318.876465:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354615ac0.
00010000:00000010:2.1:1788572318.881323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.881331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572318.888433:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:2.1:1788572318.888462:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349622100.
00010000:00000010:2.1:1788572318.896965:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.896974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572318.902600:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:2.1:1788572318.909169:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527a5280.
00010000:00000010:2.1:1788572318.918432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.918451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572318.922430:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:2.1:1788572318.922457:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494439c0.
00010000:00000010:2.1:1788572318.936482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.936497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572318.942067:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:2.1:1788572318.942088:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534639c200.
00010000:00000010:2.1:1788572318.945641:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.945658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572318.962551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:2.1:1788572318.962580:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532d6b40.
00010000:00000010:2.1:1788572318.980513:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572318.980520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572318.991476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:2.1:1788572318.991496:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534638e300.
00010000:00000010:3.1:1788572318.997761:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572318.997768:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:3.1:1788572319.008419:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:3.1:1788572319.008453:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534210a940.
00010000:00000010:3.1:1788572319.025449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572319.025457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:2.1:1788572319.038325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8da0.
00000080:00000010:2.1:1788572319.038354:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d9080.
00010000:00000010:2.1:1788572319.038363:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572319.038368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572319.046438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:2.1:1788572319.046464:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d8840.
00010000:00000010:3.1:1788572319.066688:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572319.066704:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572319.079592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:2.1:1788572319.079621:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620e300.
00010000:00000010:2.1:1788572319.089223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572319.089241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572319.091597:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421a6b40.
00000080:00000010:2.1:1788572319.117891:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00010000:00000010:2.1:1788572319.129531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572319.129539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:3.1:1788572319.139311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ba0.
00000080:00000010:3.1:1788572319.139337:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534621f380.
00010000:00000010:3.1:1788572319.139345:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572319.139350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00010000:00000010:0.1:1788572319.142927:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:0.1:1788572319.142936:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:3.1:1788572319.147330:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87e0.
00000080:00000010:3.1:1788572319.147356:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535453b180.
00000080:00000010:2.1:1788572319.151645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d82e0.
00000080:00000010:2.1:1788572319.151670:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354492940.
00010000:00000010:2.1:1788572319.155760:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572319.155780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.167147:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:2.1:1788572319.170814:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534965c200.
00010000:00000010:2.1:1788572319.176129:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572319.176143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.191591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:2.1:1788572319.191619:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346008000.
00010000:00000010:2.1:1788572319.194756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572319.194764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572319.198717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:3.1:1788572319.198746:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b30000.
00010000:00000010:3.1:1788572319.206460:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572319.206479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:2.1:1788572319.229465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d87e0.
00000080:00000010:2.1:1788572319.229491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a9b9c0.
00010000:00000010:0.1:1788572319.232648:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:0.1:1788572319.232657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.236629:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:2.1:1788572319.236658:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544a6b40.
00010000:00000010:2.1:1788572319.255436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.255457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572319.263025:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00000080:00000010:3.1:1788572319.263053:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a418c0.
00010000:00000010:3.1:1788572319.271334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572319.271342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572319.285568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83e0.
00000080:00000010:3.1:1788572319.285598:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aec200.
00010000:00000010:2.1:1788572319.300882:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.300896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00000080:00000010:2.1:1788572319.300903:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d40.
00000080:00000010:2.1:1788572319.300922:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aeeb40.
00010000:00000010:2.1:1788572319.309132:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572319.309147:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572319.313458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:2.1:1788572319.313485:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463339c0.
00010000:00000010:2.1:1788572319.321672:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.321688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.326652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:2.1:1788572319.326673:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533d4200.
00010000:00000010:2.1:1788572319.347836:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.347854:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572319.357432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:2.1:1788572319.357462:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346325280.
00010000:00000010:1.1:1788572319.357682:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572319.357690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:1.1:1788572319.365811:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb820.
00000080:00000010:1.1:1788572319.365848:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354480000.
00010000:00000010:1.1:1788572319.369993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572319.370016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572319.376573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5860.
00000080:00000010:1.1:1788572319.376603:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601ca40.
00010000:00000010:1.1:1788572319.392761:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:1.1:1788572319.392780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:1.1:1788572319.392786:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5160.
00000080:00000010:1.1:1788572319.392799:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534975c200.
00010000:00000010:0.1:1788572319.399652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074c00.
00010000:00000010:0.1:1788572319.399667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00000080:00000010:2.1:1788572319.414648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5860.
00000080:00000010:0.1:1788572319.416756:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421eb180.
00010000:00000010:0.1:1788572319.429516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572319.429534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572319.443653:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e20.
00000080:00000010:0.1:1788572319.443679:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533eeb40.
00010000:00000010:0.1:1788572319.463554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572319.463576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572319.474587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5860.
00000080:00000010:0.1:1788572319.474614:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535193a940.
00010000:00000010:0.1:1788572319.507954:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572319.507970:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572319.526701:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e20.
00000080:00000010:1.1:1788572319.527437:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519bc200.
00010000:00000010:0.1:1788572319.537085:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:0.1:1788572319.537099:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:0.1:1788572319.575449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5680.
00000080:00000010:0.1:1788572319.575468:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544d8840.
00010000:00000010:2.1:1788572319.580443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.580454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:2.1:1788572319.580458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e20.
00000080:00000010:2.1:1788572319.580472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519dd280.
00010000:00000010:2.1:1788572319.590419:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572319.590435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572319.597247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:2.1:1788572319.601548:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532618c0.
00010000:00000010:2.1:1788572319.632552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572319.632563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572319.637419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572319.637459:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533ac200.
00010000:00000010:2.1:1788572319.645683:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572319.645704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.658609:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:2.1:1788572319.658643:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b6b9c0.
00010000:00000010:2.1:1788572319.694455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572319.694462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:3.1:1788572319.704433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572319.704449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572319.705453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572319.705478:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519b1080.
00000080:00000010:3.1:1788572319.709149:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb720.
00000080:00000010:3.1:1788572319.709178:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518f5280.
00010000:00000010:2.1:1788572319.724175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572319.724194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:2.1:1788572319.737434:0:1046:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83e0.
00000080:00000010:2.1:1788572319.737466:0:1046:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953545298c0.
00010000:00000010:2.1:1788572319.737473:0:1046:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.737476:0:1046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.746479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572319.746493:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535181e300.
00010000:00000010:3.1:1788572319.753433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572319.753448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:0.1:1788572319.764988:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf20.
00000080:00000010:0.1:1788572319.765026:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb5ac0.
00010000:00000010:1.1:1788572319.773634:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:1.1:1788572319.773643:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:2.1:1788572319.778453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5dc0.
00000080:00000010:2.1:1788572319.778478:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bc6300.
00010000:00000010:2.1:1788572319.782259:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.782267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.788462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572319.788491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e4200.
00010000:00000010:2.1:1788572319.799607:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.799625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.823455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:2.1:1788572319.823477:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b95280.
00010000:00000010:2.1:1788572319.832458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.832475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.832481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572319.832497:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b96300.
00010000:00000010:2.1:1788572319.841516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.841531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572319.854914:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572319.868650:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534622eb40.
00010000:00000010:2.1:1788572319.876166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.876184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572319.888667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572319.888696:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346035280.
00010000:00000010:2.1:1788572319.896883:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572319.896898:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.900758:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572319.900773:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546cd280.
00010000:00000010:2.1:1788572319.908628:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.908635:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572319.924420:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:2.1:1788572319.924549:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a798c0.
00010000:00000010:2.1:1788572319.935778:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572319.935785:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.948494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572319.948541:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342131080.
00010000:00000010:2.1:1788572319.948553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.948558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572319.957440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572319.957460:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495fa940.
00010000:00000010:2.1:1788572319.972141:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572319.972156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572319.982638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:2.1:1788572319.982663:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a6e300.
00010000:00000010:2.1:1788572319.991432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572319.991438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572319.991456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572319.991472:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546cc200.
00010000:00000010:1.1:1788572319.996469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572319.996484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572319.996602:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572319.996617:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534952f380.
00010000:00000010:0.1:1788572320.000443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:0.1:1788572320.000461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572320.013448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:2.1:1788572320.013480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e39c0.
00010000:00000010:0.1:1788572320.033481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572320.033502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572320.045757:0:1377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572320.045778:0:1377:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346068840.
00010000:00000010:0.1:1788572320.050185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:0.1:1788572320.050201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572320.057989:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:2.1:1788572320.058024:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463aca40.
00010000:00000010:2.1:1788572320.075211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.075219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572320.087185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:2.1:1788572320.087202:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346036300.
00010000:00000010:2.1:1788572320.094588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.094604:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572320.094610:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:2.1:1788572320.094625:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aa3180.
00010000:00000010:2.1:1788572320.103600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.103617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572320.126440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572320.126467:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535330dac0.
00010000:00000010:2.1:1788572320.135650:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.135659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572320.157438:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:2.1:1788572320.157533:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526aca40.
00010000:00000010:2.1:1788572320.165537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.165549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572320.165554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572320.165570:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346034a40.
00010000:00000010:1.1:1788572320.168804:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:1.1:1788572320.168812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:0.1:1788572320.174278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00000080:00000010:0.1:1788572320.177925:0:2354:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495f8000.
00010000:00000010:1.1:1788572320.192417:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572320.192425:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:1.1:1788572320.207592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa50c0.
00000080:00000010:1.1:1788572320.207610:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535256a100.
00010000:00000010:1.1:1788572320.222269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572320.222287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572320.222293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d60.
00000080:00000010:1.1:1788572320.222310:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346014200.
00010000:00000010:1.1:1788572320.230516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572320.230533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:1.1:1788572320.244599:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5580.
00000080:00000010:1.1:1788572320.244627:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346328000.
00010000:00000010:1.1:1788572320.257923:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572320.257941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572320.266436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d60.
00000080:00000010:1.1:1788572320.266464:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534621c200.
00010000:00000010:1.1:1788572320.281173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572320.281189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:1.1:1788572320.281195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ac0.
00000080:00000010:1.1:1788572320.281210:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497ab180.
00010000:00000010:1.1:1788572320.291795:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572320.291809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572320.291816:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5580.
00000080:00000010:1.1:1788572320.291834:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497518c0.
00010000:00000010:0.1:1788572320.300564:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572320.300574:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:0.1:1788572320.309638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d60.
00000080:00000010:0.1:1788572320.309665:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421eb9c0.
00010000:00000010:0.1:1788572320.318601:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572320.318609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572320.321707:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a60.
00000080:00000010:0.1:1788572320.321725:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b44a40.
00010000:00000010:0.1:1788572320.338551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572320.338559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572320.338564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5400.
00000080:00000010:0.1:1788572320.338593:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342178840.
00010000:00000010:1.1:1788572320.353141:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572320.353149:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:2.1:1788572320.358451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a60.
00000080:00000010:2.1:1788572320.358476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351982940.
00010000:00000010:2.1:1788572320.381557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.381565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572320.390555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572320.390575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533c0000.
00010000:00000010:1.1:1788572320.397604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572320.397620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:1.1:1788572320.404695:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc20.
00000080:00000010:1.1:1788572320.404742:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519de300.
00000080:00000010:2.1:1788572320.414143:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ee0.
00000080:00000010:2.1:1788572320.414169:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b6300.
00010000:00000010:0.1:1788572320.414207:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:0.1:1788572320.414214:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00010000:00000010:2.1:1788572320.423623:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.423633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572320.428276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572320.428298:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535270f380.
00010000:00000010:2.1:1788572320.432452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.432466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572320.442118:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572320.442143:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349678000.
00010000:00000010:2.1:1788572320.453448:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.453455:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572320.458450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:2.1:1788572320.462211:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535193b9c0.
00010000:00000010:0.1:1788572320.471714:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572320.471730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572320.483759:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572320.483787:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354525ac0.
00010000:00000010:0.1:1788572320.501162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:0.1:1788572320.501171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:0.1:1788572320.520016:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb660.
00000080:00000010:0.1:1788572320.520141:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519b6b40.
00010000:00000010:0.1:1788572320.520167:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572320.520172:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572320.532465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5840.
00000080:00000010:0.1:1788572320.532486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535452b180.
00010000:00000010:0.1:1788572320.543780:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:0.1:1788572320.543788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:0.1:1788572320.549463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c80.
00000080:00000010:0.1:1788572320.549491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bc5280.
00010000:00000010:1.1:1788572320.561717:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4b00.
00010000:00000010:1.1:1788572320.561735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572320.585506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5840.
00000080:00000010:0.1:1788572320.585531:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354544a40.
00010000:00000010:1.1:1788572320.590452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572320.590466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:2.1:1788572320.593558:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5640.
00000080:00000010:2.1:1788572320.593577:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535454ca40.
00010000:00000010:2.1:1788572320.607474:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572320.607490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:2.1:1788572320.616692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572320.616723:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b6a940.
00010000:00000010:0.1:1788572320.631992:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572320.632009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:0.1:1788572320.638548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbce0.
00000080:00000010:0.1:1788572320.638575:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534230dac0.
00010000:00000010:0.1:1788572320.647661:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572320.647677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572320.656956:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5700.
00000080:00000010:0.1:1788572320.656997:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535324b180.
00010000:00000010:0.1:1788572320.679590:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:0.1:1788572320.679597:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:0.1:1788572320.698566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f60.
00000080:00000010:0.1:1788572320.698584:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544b6b40.
00010000:00000010:3.1:1788572320.715127:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572320.715134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:2.1:1788572320.716987:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5700.
00000080:00000010:2.1:1788572320.717008:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463cca40.
00010000:00000010:2.1:1788572320.721270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:2.1:1788572320.721278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572320.724634:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572320.724671:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421498c0.
00010000:00000010:2.1:1788572320.730556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:2.1:1788572320.730574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572320.738671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:2.1:1788572320.738687:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352700840.
00010000:00000010:2.1:1788572320.749567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:2.1:1788572320.749577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572320.760704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572320.760728:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352638840.
00010000:00000010:2.1:1788572320.775459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:2.1:1788572320.775470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572320.781226:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463ce300.
00000080:00000010:2.1:1788572320.783435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00010000:00000010:3.1:1788572320.789441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:3.1:1788572320.789459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572320.805474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5620.
00000080:00000010:0.1:1788572320.805525:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495de300.
00010000:00000010:1.1:1788572320.820659:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572320.820674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:0.1:1788572320.828507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53c0.
00000080:00000010:0.1:1788572320.828536:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494fc200.
00010000:00000010:2.1:1788572320.844713:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4e00.
00010000:00000010:2.1:1788572320.844727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:2.1:1788572320.868712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a20.
00000080:00000010:2.1:1788572320.868743:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519a0840.
00010000:00000010:2.1:1788572320.883455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572320.883464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572320.883470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:2.1:1788572320.883497:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535443a940.
00010000:00000010:2.1:1788572320.900843:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572320.900859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572320.909144:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572320.909171:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f4a40.
00010000:00000010:2.1:1788572320.916607:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572320.916625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572320.916632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:2.1:1788572320.916650:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a2100.
00010000:00000010:2.1:1788572320.922557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572320.922571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572320.927466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572320.927495:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497a1080.
00010000:00000010:2.1:1788572320.935542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572320.935556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572320.945989:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:2.1:1788572320.946036:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b79080.
00010000:00000010:2.1:1788572320.957463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572320.957478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572320.974621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572320.974662:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ac4a40.
00010000:00000010:1.1:1788572320.978658:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572320.978672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:1.1:1788572320.989520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb020.
00000080:00000010:1.1:1788572320.989552:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494eb9c0.
00010000:00000010:1.1:1788572321.006626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572321.006644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572321.010632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5900.
00000080:00000010:1.1:1788572321.010662:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a4f380.
00010000:00000010:1.1:1788572321.016433:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572321.016440:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:1.1:1788572321.021455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5320.
00000080:00000010:1.1:1788572321.025805:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546e6b40.
00010000:00000010:1.1:1788572321.030462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572321.030478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:1.1:1788572321.049421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5900.
00000080:00000010:1.1:1788572321.049457:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421ba940.
00010000:00000010:2.1:1788572321.054538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572321.054556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:3.1:1788572321.069492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5320.
00000080:00000010:3.1:1788572321.069533:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b29080.
00010000:00000010:3.1:1788572321.078858:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572321.078883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572321.095721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8500.
00000080:00000010:3.1:1788572321.095752:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e2100.
00010000:00000010:3.1:1788572321.100516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572321.100531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572321.107480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d83a0.
00000080:00000010:3.1:1788572321.107517:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b26b40.
00010000:00000010:2.1:1788572321.125529:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:2.1:1788572321.125537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:2.1:1788572321.130336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8500.
00000080:00000010:2.1:1788572321.130367:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474f380.
00010000:00000010:2.1:1788572321.134360:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:2.1:1788572321.134376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.147752:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572321.147778:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533aa100.
00010000:00000010:2.1:1788572321.155591:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572321.155607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.176775:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572321.176804:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497a7380.
00010000:00000010:3.1:1788572321.200437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572321.200460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.204422:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572321.204976:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a18000.
00010000:00000010:2.1:1788572321.211578:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572321.211596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572321.221785:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572321.221815:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519039c0.
00010000:00000010:2.1:1788572321.228611:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572321.228621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572321.244725:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572321.254602:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346046b40.
00010000:00000010:2.1:1788572321.259500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572321.259508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572321.270294:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572321.270331:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527e0840.
00010000:00000010:2.1:1788572321.292343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572321.292360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:1.1:1788572321.302608:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4f00.
00010000:00000010:1.1:1788572321.302615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:2.1:1788572321.302974:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572321.302992:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a54a40.
00000080:00000010:2.1:1788572321.303019:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:2.1:1788572321.303032:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953462098c0.
00010000:00000010:3.1:1788572321.312434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572321.312500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:3.1:1788572321.332022:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb060.
00000080:00000010:3.1:1788572321.332598:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a24200.
00010000:00000010:3.1:1788572321.337924:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572321.337932:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:2.1:1788572321.350986:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c40.
00000080:00000010:2.1:1788572321.351012:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351914200.
00010000:00000010:2.1:1788572321.370282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572321.370298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:2.1:1788572321.378721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572321.378738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:2.1:1788572321.378745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5be0.
00000080:00000010:2.1:1788572321.378775:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422fc200.
00000080:00000010:1.1:1788572321.378830:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd20.
00000080:00000010:1.1:1788572321.378845:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354532100.
00010000:00000010:2.1:1788572321.389464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572321.389471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572321.406228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:3.1:1788572321.406848:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953526e39c0.
00000080:00000010:0.1:1788572321.425433:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8d60.
00010000:00000010:2.1:1788572321.425591:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:2.1:1788572321.425597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:0.1:1788572321.425753:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b19080.
00010000:00000010:2.1:1788572321.439642:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572321.439652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:2.1:1788572321.439657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5820.
00000080:00000010:2.1:1788572321.439686:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351911080.
00010000:00000010:0.1:1788572321.443464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572321.443481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572321.447186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572321.447219:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534947d280.
00010000:00000010:2.1:1788572321.452510:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572321.452529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.458551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572321.458587:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a87380.
00010000:00000010:2.1:1788572321.475542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572321.475556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.476452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:2.1:1788572321.476475:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546feb40.
00010000:00000010:2.1:1788572321.484219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572321.484227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572321.494459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572321.494476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bd7380.
00010000:00000010:2.1:1788572321.507902:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572321.507912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:0.1:1788572321.508131:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb520.
00000080:00000010:0.1:1788572321.508151:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463d5280.
00010000:00000010:3.1:1788572321.526664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:3.1:1788572321.526673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:2.1:1788572321.551427:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e00.
00000080:00000010:2.1:1788572321.551455:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b81080.
00010000:00000010:0.1:1788572321.564875:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:0.1:1788572321.564901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572321.565257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572321.565303:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346225280.
00010000:00000010:2.1:1788572321.565311:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572321.565315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.573432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:2.1:1788572321.573481:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f5ac0.
00010000:00000010:2.1:1788572321.578424:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572321.578431:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:2.1:1788572321.600855:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:2.1:1788572321.600895:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a3180.
00010000:00000010:0.1:1788572321.605767:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572321.605777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:0.1:1788572321.615663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00000080:00000010:0.1:1788572321.615701:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544ba100.
00010000:00000010:0.1:1788572321.622349:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:0.1:1788572321.622357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572321.637009:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c40.
00000080:00000010:0.1:1788572321.637024:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349660000.
00010000:00000010:2.1:1788572321.645623:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4900.
00010000:00000010:2.1:1788572321.645632:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98800.
00000080:00000010:2.1:1788572321.666449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5080.
00000080:00000010:2.1:1788572321.666482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535322a100.
00010000:00000010:2.1:1788572321.673438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572321.673461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.673467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:2.1:1788572321.673497:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ad4a40.
00010000:00000010:0.1:1788572321.679567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572321.679577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572321.690618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572321.690658:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533e0840.
00010000:00000010:2.1:1788572321.694656:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572321.694672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.706226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:2.1:1788572321.706263:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544f4200.
00010000:00000010:2.1:1788572321.724784:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572321.724792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.728873:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572321.728914:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342158000.
00010000:00000010:2.1:1788572321.746460:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572321.746476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572321.750456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:2.1:1788572321.750530:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ad18c0.
00010000:00000010:2.1:1788572321.750538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572321.750543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.760521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572321.760546:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519ea100.
00010000:00000010:0.1:1788572321.784492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572321.784508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:2.1:1788572321.793438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:2.1:1788572321.793468:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494b6300.
00010000:00000010:0.1:1788572321.802491:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:0.1:1788572321.802507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572321.813550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572321.813571:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346348000.
00010000:00000010:2.1:1788572321.817991:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572321.818007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572321.821618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:2.1:1788572321.821658:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349544200.
00010000:00000010:2.1:1788572321.832501:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572321.832514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572321.839414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:2.1:1788572321.839471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354765ac0.
00010000:00000010:2.1:1788572321.862828:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572321.862836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572321.865571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:2.1:1788572321.865591:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a3d280.
00010000:00000010:3.1:1788572321.883983:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572321.883989:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572321.891492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00000080:00000010:3.1:1788572321.891535:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342312100.
00010000:00000010:2.1:1788572321.895431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572321.895441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:0.1:1788572321.895453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8260.
00010000:00000010:2.1:1788572321.899752:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572321.899768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:0.1:1788572321.901059:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b9c200.
00000080:00000010:2.1:1788572321.912639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa54c0.
00000080:00000010:2.1:1788572321.912665:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535192b180.
00010000:00000010:3.1:1788572321.942945:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572321.942960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:3.1:1788572321.950849:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba20.
00000080:00000010:3.1:1788572321.950874:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535329a100.
00010000:00000010:2.1:1788572321.980467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572321.980480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:2.1:1788572321.984709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8fe0.
00000080:00000010:2.1:1788572321.984737:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463e9080.
00010000:00000010:2.1:1788572321.995445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572321.995463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.013223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0c0.
00000080:00000010:2.1:1788572322.013259:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bfdac0.
00010000:00000010:2.1:1788572322.023597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.023612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572322.038718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba80.
00000080:00000010:3.1:1788572322.038748:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534958c200.
00010000:00000010:1.1:1788572322.043626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572322.043634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:1.1:1788572322.053026:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8120.
00000080:00000010:1.1:1788572322.053053:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535447eb40.
00010000:00000010:3.1:1788572322.060551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572322.060565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:3.1:1788572322.074657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5840.
00000080:00000010:3.1:1788572322.082625:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535269b9c0.
00010000:00000010:3.1:1788572322.089454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572322.089467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:2.1:1788572322.096268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8ce0.
00000080:00000010:2.1:1788572322.096321:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351992100.
00010000:00000010:2.1:1788572322.097474:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.097485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.106158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0c0.
00000080:00000010:2.1:1788572322.106178:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351977380.
00010000:00000010:2.1:1788572322.117246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.117255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.135287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:2.1:1788572322.135346:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352755ac0.
00010000:00000010:2.1:1788572322.135353:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.135358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572322.143456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb0c0.
00000080:00000010:3.1:1788572322.143482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527d1080.
00010000:00000010:3.1:1788572322.152709:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572322.152724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572322.159429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:3.1:1788572322.159470:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495d2940.
00010000:00000010:3.1:1788572322.171452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572322.171468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:2.1:1788572322.180449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8980.
00000080:00000010:2.1:1788572322.180474:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518d0840.
00010000:00000010:2.1:1788572322.202953:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.202962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.207454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:2.1:1788572322.207493:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535446e300.
00010000:00000010:2.1:1788572322.216445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.216460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572322.229767:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.229801:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351989080.
00010000:00000010:2.1:1788572322.238886:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.238909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572322.243455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:2.1:1788572322.243484:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b52100.
00010000:00000010:2.1:1788572322.252609:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.252624:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.273043:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.273070:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534974a940.
00010000:00000010:2.1:1788572322.278810:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.278818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.286818:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:2.1:1788572322.286835:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421e2100.
00010000:00000010:2.1:1788572322.307464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.307481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572322.307487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.311573:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349748000.
00010000:00000010:2.1:1788572322.330479:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.330497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.330503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:2.1:1788572322.330521:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533df380.
00010000:00000010:2.1:1788572322.344458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572322.344475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572322.354605:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.354637:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346338840.
00010000:00000010:3.1:1788572322.375040:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:3.1:1788572322.375057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:3.1:1788572322.392447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00000080:00000010:3.1:1788572322.397480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534218b9c0.
00010000:00000010:3.1:1788572322.421639:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572322.421647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572322.427188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:3.1:1788572322.427228:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354476b40.
00010000:00000010:3.1:1788572322.442732:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572322.442748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572322.455437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8160.
00000080:00000010:3.1:1788572322.455461:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bbb180.
00010000:00000010:0.1:1788572322.469470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572322.469478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572322.495457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e40.
00000080:00000010:0.1:1788572322.495502:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354472100.
00010000:00000010:0.1:1788572322.515155:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572322.515162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572322.518560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5800.
00000080:00000010:0.1:1788572322.518586:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535269a940.
00010000:00000010:0.1:1788572322.536574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572322.536588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99e00.
00000080:00000010:2.1:1788572322.575432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e40.
00000080:00000010:2.1:1788572322.575557:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354419080.
00010000:00000010:2.1:1788572322.582612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572322.582627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.598694:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.598711:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534958f380.
00010000:00000010:0.1:1788572322.599868:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572322.599879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:0.1:1788572322.603595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb920.
00000080:00000010:0.1:1788572322.603623:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bb8000.
00010000:00000010:0.1:1788572322.615727:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572322.615744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572322.634438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5820.
00000080:00000010:0.1:1788572322.634491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353263180.
00010000:00000010:0.1:1788572322.637789:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572322.637798:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:0.1:1788572322.647460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e80.
00000080:00000010:0.1:1788572322.651469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544d0840.
00010000:00000010:2.1:1788572322.651546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572322.651555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:2.1:1788572322.671745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f20.
00000080:00000010:2.1:1788572322.671762:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421d6300.
00010000:00000010:2.1:1788572322.675877:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572322.675897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.703436:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.703462:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535197d280.
00010000:00000010:2.1:1788572322.713658:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572322.713665:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.713670:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572322.713683:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352750840.
00010000:00000010:2.1:1788572322.724433:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.724439:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572322.740639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbec0.
00000080:00000010:2.1:1788572322.740662:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421e4200.
00010000:00000010:2.1:1788572322.750339:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.750354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.758175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572322.758200:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953497498c0.
00010000:00000010:2.1:1788572322.763001:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.763009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:1.1:1788572322.773431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbec0.
00000080:00000010:1.1:1788572322.773477:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342232100.
00010000:00000010:3.1:1788572322.787756:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572322.787771:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28000.
00000080:00000010:3.1:1788572322.797843:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53a0.
00000080:00000010:3.1:1788572322.797876:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421af380.
00010000:00000010:2.1:1788572322.801424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.801431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572322.805641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8160.
00000080:00000010:2.1:1788572322.805686:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b598c0.
00010000:00000010:2.1:1788572322.823415:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.823431:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.835848:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572322.835867:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533dc200.
00010000:00000010:2.1:1788572322.839993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.840010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572322.840016:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.840049:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346334a40.
00010000:00000010:2.1:1788572322.860343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.860357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.860406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb880.
00000080:00000010:2.1:1788572322.860439:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354732940.
00010000:00000010:2.1:1788572322.877438:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.877456:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572322.885478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.885501:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a62100.
00010000:00000010:2.1:1788572322.889436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.889454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.893452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb880.
00000080:00000010:2.1:1788572322.893481:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ad0000.
00010000:00000010:2.1:1788572322.903547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.903565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572322.926485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.934521:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420bf380.
00010000:00000010:2.1:1788572322.943427:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.943434:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572322.951478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb880.
00000080:00000010:2.1:1788572322.951516:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534623e300.
00010000:00000010:2.1:1788572322.959900:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572322.959916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572322.964453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572322.964483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346342940.
00010000:00000010:0.1:1788572322.990438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:0.1:1788572322.990596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:0.1:1788572322.998518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb880.
00000080:00000010:0.1:1788572322.998536:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534215a940.
00010000:00000010:0.1:1788572322.998563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572322.998568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572323.004372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5920.
00000080:00000010:0.1:1788572323.004418:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535337dac0.
00010000:00000010:0.1:1788572323.014031:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572323.014046:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98400.
00000080:00000010:0.1:1788572323.024214:0:2355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa51e0.
00000080:00000010:0.1:1788572323.024243:0:2355:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953494b5280.
00010000:00000010:1.1:1788572323.028438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572323.028457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99800.
00000080:00000010:0.1:1788572323.040092:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5920.
00000080:00000010:0.1:1788572323.040140:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b9f380.
00010000:00000010:0.1:1788572323.044505:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572323.044519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:2.1:1788572323.052686:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa51e0.
00000080:00000010:2.1:1788572323.052729:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346314200.
00010000:00000010:2.1:1788572323.059657:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.059673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572323.069744:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.073471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463a18c0.
00010000:00000010:2.1:1788572323.080997:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.081015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572323.089434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572323.089460:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535329eb40.
00010000:00000010:2.1:1788572323.101264:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.101273:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572323.106541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.106585:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346008840.
00010000:00000010:2.1:1788572323.120822:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.120832:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572323.134452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572323.134490:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b3b9c0.
00010000:00000010:2.1:1788572323.137576:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.137583:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572323.141618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.141657:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a99080.
00010000:00000010:2.1:1788572323.161600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.161615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572323.165821:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572323.165839:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346058840.
00010000:00000010:2.1:1788572323.171634:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.171650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572323.181677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.181726:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b3ca40.
00010000:00000010:2.1:1788572323.189794:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.189803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572323.211951:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572323.211989:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518098c0.
00010000:00000010:2.1:1788572323.221423:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.221429:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572323.221433:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.230469:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354559080.
00010000:00000010:2.1:1788572323.240821:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572323.240835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572323.255459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572323.255478:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ac0000.
00010000:00000010:2.1:1788572323.255487:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572323.255492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572323.259450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.259491:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ae18c0.
00010000:00000010:3.1:1788572323.272118:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:3.1:1788572323.272136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572323.284712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8a0.
00000080:00000010:3.1:1788572323.284741:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535190b9c0.
00010000:00000010:3.1:1788572323.296293:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:3.1:1788572323.296307:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572323.311349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:3.1:1788572323.311374:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527a2940.
00010000:00000010:2.1:1788572323.332410:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abf00.
00010000:00000010:2.1:1788572323.332428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:2.1:1788572323.355439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8b60.
00000080:00000010:2.1:1788572323.355543:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534944a100.
00010000:00000010:2.1:1788572323.363434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.363449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572323.370853:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953532d2100.
00000080:00000010:2.1:1788572323.372386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.372448:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b5d280.
00010000:00000010:2.1:1788572323.372466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.372472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.372476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00010000:00000010:3.1:1788572323.391017:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572323.391031:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572323.391036:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:3.1:1788572323.391060:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534638eb40.
00010000:00000010:3.1:1788572323.400851:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572323.400860:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572323.400865:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572323.400879:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342108000.
00010000:00000010:3.1:1788572323.412833:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572323.412845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:3.1:1788572323.423612:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:0.1:1788572323.423776:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463deb40.
00010000:00000010:0.1:1788572323.434900:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572323.434912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99a00.
00000080:00000010:0.1:1788572323.442770:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5dc0.
00000080:00000010:0.1:1788572323.442790:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d4a40.
00010000:00000010:0.1:1788572323.453026:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572323.453035:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:0.1:1788572323.458284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5200.
00000080:00000010:0.1:1788572323.463679:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346301080.
00010000:00000010:2.1:1788572323.478454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.478465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00000080:00000010:2.1:1788572323.490658:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ee0.
00000080:00000010:2.1:1788572323.490672:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953496318c0.
00010000:00000010:2.1:1788572323.508066:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.508083:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.508088:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:2.1:1788572323.508217:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463fa100.
00010000:00000010:2.1:1788572323.508229:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572323.508232:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572323.544453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.544480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534234e300.
00010000:00000010:1.1:1788572323.547901:0:2354:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:1.1:1788572323.547910:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:3.1:1788572323.552682:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00000080:00000010:3.1:1788572323.552711:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463fb180.
00010000:00000010:0.1:1788572323.556438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572323.556452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00000080:00000010:1.1:1788572323.564763:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8420.
00000080:00000010:1.1:1788572323.564793:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953423098c0.
00010000:00000010:0.1:1788572323.574463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572323.574483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29a00.
00000080:00000010:0.1:1788572323.584677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d60.
00000080:00000010:0.1:1788572323.584725:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a8a940.
00010000:00000010:0.1:1788572323.594044:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:0.1:1788572323.594059:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:0.1:1788572323.621561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa59a0.
00000080:00000010:0.1:1788572323.621577:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895353242940.
00010000:00000010:1.1:1788572323.622493:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4100.
00010000:00000010:1.1:1788572323.622503:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98200.
00000080:00000010:1.1:1788572323.627471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5740.
00000080:00000010:1.1:1788572323.635585:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546fca40.
00010000:00000010:1.1:1788572323.644595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:1.1:1788572323.644610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:0.1:1788572323.647692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5480.
00000080:00000010:0.1:1788572323.647739:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421098c0.
00010000:00000010:0.1:1788572323.658182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:0.1:1788572323.658195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00000080:00000010:2.1:1788572323.666090:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53e0.
00000080:00000010:2.1:1788572323.666127:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346306b40.
00010000:00000010:2.1:1788572323.676459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.676474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.705408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.705457:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422d1080.
00010000:00000010:2.1:1788572323.715690:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.715699:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.724805:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:2.1:1788572323.724842:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535268a940.
00010000:00000010:0.1:1788572323.748086:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572323.748100:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99200.
00000080:00000010:2.1:1788572323.750454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.750494:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346396b40.
00010000:00000010:2.1:1788572323.756047:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.756058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.756063:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:2.1:1788572323.756090:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346387380.
00010000:00000010:2.1:1788572323.760445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572323.760460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572323.774623:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb9e0.
00000080:00000010:3.1:1788572323.774642:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534234ca40.
00010000:00000010:3.1:1788572323.783611:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572323.783618:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:3.1:1788572323.799858:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:3.1:1788572323.799887:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349590000.
00010000:00000010:2.1:1788572323.810692:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:2.1:1788572323.810701:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00000080:00000010:2.1:1788572323.810706:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8100.
00000080:00000010:2.1:1788572323.828678:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349630000.
00010000:00000010:2.1:1788572323.836481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572323.836494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572323.847618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:2.1:1788572323.847660:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463c1080.
00010000:00000010:2.1:1788572323.848621:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.848631:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.861378:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.861434:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342085280.
00010000:00000010:2.1:1788572323.880520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.880532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572323.880537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:2.1:1788572323.880565:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953495a8000.
00010000:00000010:2.1:1788572323.887627:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572323.887638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.887643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.887657:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354616300.
00010000:00000010:2.1:1788572323.892462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.892480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572323.912535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:2.1:1788572323.912579:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b51080.
00000080:00000010:2.1:1788572323.923703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.923737:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534620dac0.
00010000:00000010:0.1:1788572323.924439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572323.924456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:2.1:1788572323.928456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.928472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.935740:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:2.1:1788572323.935770:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953546fdac0.
00010000:00000010:2.1:1788572323.942639:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.942656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.957210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572323.957229:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534212e300.
00010000:00000010:2.1:1788572323.965213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.965232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572323.981446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:2.1:1788572323.981471:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a80000.
00010000:00000010:2.1:1788572323.989586:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572323.989604:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572324.006597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572324.006622:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421b98c0.
00010000:00000010:2.1:1788572324.018572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572324.018589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572324.029883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:2.1:1788572324.029903:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349562940.
00010000:00000010:2.1:1788572324.038580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572324.038598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572324.063826:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:2.1:1788572324.063856:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535455eb40.
00010000:00000010:2.1:1788572324.088597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572324.088613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572324.096143:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:2.1:1788572324.096173:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b0b9c0.
00010000:00000010:3.1:1788572324.104437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572324.104461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:3.1:1788572324.132829:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:3.1:1788572324.132857:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535474d280.
00010000:00000010:2.1:1788572324.150539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:2.1:1788572324.150552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00000080:00000010:2.1:1788572324.159921:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8c60.
00000080:00000010:2.1:1788572324.166628:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953422fd280.
00010000:00000010:2.1:1788572324.181512:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572324.181528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572324.201262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:2.1:1788572324.201315:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b05280.
00010000:00000010:2.1:1788572324.222455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572324.222471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572324.242536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:2.1:1788572324.242573:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463b98c0.
00010000:00000010:3.1:1788572324.252767:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:3.1:1788572324.252783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:3.1:1788572324.275572:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:3.1:1788572324.275599:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535451a100.
00010000:00000010:3.1:1788572324.300562:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:3.1:1788572324.300571:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:3.1:1788572324.307858:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d88c0.
00000080:00000010:3.1:1788572324.307896:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354493180.
00010000:00000010:2.1:1788572324.318460:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:2.1:1788572324.318474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:2.1:1788572324.326047:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8be0.
00000080:00000010:2.1:1788572324.326086:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ac18c0.
00010000:00000010:2.1:1788572324.340456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.340472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572324.367753:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:2.1:1788572324.367792:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351900000.
00010000:00000010:2.1:1788572324.375634:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.375650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572324.398557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:2.1:1788572324.398591:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a10840.
00010000:00000010:2.1:1788572324.407559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.407575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572324.414979:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:2.1:1788572324.415019:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534975e300.
00010000:00000010:2.1:1788572324.422597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.422611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:1.1:1788572324.437684:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00000080:00000010:1.1:1788572324.437704:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b41080.
00010000:00000010:1.1:1788572324.451777:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572324.451818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:1.1:1788572324.457482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5620.
00000080:00000010:1.1:1788572324.457505:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534601f380.
00010000:00000010:1.1:1788572324.464723:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:1.1:1788572324.464736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:1.1:1788572324.472063:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5560.
00000080:00000010:1.1:1788572324.472089:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953460118c0.
00010000:00000010:0.1:1788572324.488539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074d00.
00010000:00000010:0.1:1788572324.488549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:2.1:1788572324.491726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5620.
00000080:00000010:2.1:1788572324.491763:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346216b40.
00010000:00000010:2.1:1788572324.503628:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.503644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572324.519264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:2.1:1788572324.522671:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533d6300.
00010000:00000010:2.1:1788572324.534597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.534611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572324.538733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:2.1:1788572324.538763:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352698840.
00010000:00000010:2.1:1788572324.546317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.546331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:0.1:1788572324.575448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6e0.
00000080:00000010:0.1:1788572324.575473:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351818840.
00010000:00000010:0.1:1788572324.588831:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572324.588849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572324.607439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5200.
00000080:00000010:0.1:1788572324.607468:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533b7380.
00010000:00000010:0.1:1788572324.630538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572324.630554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:0.1:1788572324.661358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5dc0.
00000080:00000010:0.1:1788572324.661409:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354546300.
00010000:00000010:2.1:1788572324.672935:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.672943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98c00.
00000080:00000010:2.1:1788572324.679312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5200.
00000080:00000010:2.1:1788572324.682453:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bf3180.
00010000:00000010:2.1:1788572324.711056:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.711074:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572324.717683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:2.1:1788572324.717713:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351b818c0.
00010000:00000010:2.1:1788572324.726655:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.726670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572324.765182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbe0.
00000080:00000010:2.1:1788572324.765205:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533bca40.
00010000:00000010:2.1:1788572324.783952:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.783964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572324.788700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:2.1:1788572324.788736:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352741080.
00010000:00000010:2.1:1788572324.794526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572324.794539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572324.803459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbe0.
00000080:00000010:2.1:1788572324.803490:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953518d6300.
00010000:00000010:2.1:1788572324.807585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572324.807600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572324.815933:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:2.1:1788572324.815950:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953533bdac0.
00010000:00000010:2.1:1788572324.824534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572324.824552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:0.1:1788572324.849964:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbbe0.
00000080:00000010:0.1:1788572324.850018:0:13:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953496798c0.
00010000:00000010:2.1:1788572324.853725:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572324.853732:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99000.
00000080:00000010:2.1:1788572324.853737:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5620.
00000080:00000010:2.1:1788572324.853769:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953527418c0.
00010000:00000010:2.1:1788572324.879484:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.879497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572324.879502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:2.1:1788572324.879517:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354520840.
00010000:00000010:2.1:1788572324.883652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572324.883660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572324.883666:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:2.1:1788572324.883693:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463eca40.
00010000:00000010:2.1:1788572324.890530:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572324.890543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572324.896711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:2.1:1788572324.896744:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346066300.
00010000:00000010:3.1:1788572324.933452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572324.933470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572324.933476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe60.
00000080:00000010:3.1:1788572324.933492:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a7b9c0.
00010000:00000010:3.1:1788572324.939689:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572324.939704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:3.1:1788572324.939709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d88c0.
00000080:00000010:3.1:1788572324.939737:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535337a100.
00010000:00000010:3.1:1788572324.943567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:3.1:1788572324.943583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:2.1:1788572324.950943:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8f00.
00000080:00000010:2.1:1788572324.950967:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351af1080.
00010000:00000010:2.1:1788572324.969644:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572324.969651:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:3.1:1788572324.980839:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb260.
00000080:00000010:3.1:1788572324.980854:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346225ac0.
00010000:00000010:2.1:1788572324.995799:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572324.995807:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:2.1:1788572325.002783:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d88c0.
00000080:00000010:2.1:1788572325.002812:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346229080.
00010000:00000010:0.1:1788572325.011564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572325.011580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:0.1:1788572325.014608:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:0.1:1788572325.014636:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895352615280.
00010000:00000010:0.1:1788572325.022535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572325.022542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572325.026645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f40.
00000080:00000010:0.1:1788572325.026673:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534631a100.
00010000:00000010:0.1:1788572325.035635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572325.035653:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00000080:00000010:0.1:1788572325.051585:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa50e0.
00000080:00000010:0.1:1788572325.051603:0:13:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953463aa940.
00000080:00000010:3.1:1788572325.062661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5f40.
00000080:00000010:3.1:1788572325.062681:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535256f380.
00010000:00000010:1.1:1788572325.062740:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:1.1:1788572325.062747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00010000:00000010:1.1:1788572325.078843:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:1.1:1788572325.078851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98000.
00010000:00000010:0.1:1788572325.079433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:0.1:1788572325.079456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00000080:00000010:0.1:1788572325.079461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d80a0.
00000080:00000010:0.1:1788572325.079486:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a66300.
00000080:00000010:2.1:1788572325.083429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa54e0.
00000080:00000010:2.1:1788572325.083479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351aa7380.
00010000:00000010:2.1:1788572325.093485:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572325.093501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572325.104740:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572325.104757:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351960000.
00010000:00000010:2.1:1788572325.129580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572325.129588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572325.134349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6a0.
00000080:00000010:2.1:1788572325.134420:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a67380.
00010000:00000010:2.1:1788572325.137957:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572325.137967:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572325.143760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572325.143780:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342314a40.
00010000:00000010:0.1:1788572325.158620:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572325.158639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:1.1:1788572325.176704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6a0.
00000080:00000010:1.1:1788572325.184126:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351966b40.
00010000:00000010:2.1:1788572325.193717:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:2.1:1788572325.193725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98a00.
00000080:00000010:2.1:1788572325.197464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50a0.
00000080:00000010:2.1:1788572325.197492:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351af7380.
00010000:00000010:0.1:1788572325.207616:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56600.
00010000:00000010:0.1:1788572325.207630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00000080:00000010:2.1:1788572325.219493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572325.219545:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354489080.
00010000:00000010:2.1:1788572325.230473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572325.230487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572325.248423:0:206932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:2.1:1788572325.248458:0:206932:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346019080.
00010000:00000010:2.1:1788572325.253674:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572325.253689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572325.269654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572325.269694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953421e18c0.
00010000:00000010:3.1:1788572325.285463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572325.285478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:3.1:1788572325.297354:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00000080:00000010:3.1:1788572325.300997:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534223eb40.
00010000:00000010:2.1:1788572325.319640:0:206932:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572325.319647:0:206932:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:2.1:1788572325.332015:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953449d8e60.
00000080:00000010:2.1:1788572325.332033:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534223f380.
00010000:00000010:2.1:1788572325.347676:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572325.347685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572325.356920:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572325.356955:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bc98c0.
00010000:00000010:1.1:1788572325.368887:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:1.1:1788572325.368905:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:0.1:1788572325.378107:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf40.
00000080:00000010:0.1:1788572325.378148:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bbf380.
00010000:00000010:0.1:1788572325.389631:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572325.389646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572325.401450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5060.
00000080:00000010:0.1:1788572325.401479:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354545280.
00010000:00000010:0.1:1788572325.409124:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:0.1:1788572325.409139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:0.1:1788572325.417977:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa58a0.
00000080:00000010:0.1:1788572325.418020:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895349588000.
00010000:00000010:1.1:1788572325.431658:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4600.
00010000:00000010:1.1:1788572325.431674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99400.
00000080:00000010:2.1:1788572325.442430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5060.
00000080:00000010:2.1:1788572325.442476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895342335280.
00010000:00000010:2.1:1788572325.458473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572325.458488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572325.467094:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:2.1:1788572325.467126:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895354528000.
00010000:00000010:2.1:1788572325.474552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572325.474567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572325.481714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:2.1:1788572325.481751:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535181d280.
00010000:00000010:0.1:1788572325.499930:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572325.499938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:0.1:1788572325.507624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5c0.
00000080:00000010:0.1:1788572325.507663:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351ba8000.
00010000:00000010:0.1:1788572325.516064:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:0.1:1788572325.516072:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:0.1:1788572325.522676:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa51a0.
00000080:00000010:0.1:1788572325.527717:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351bc4a40.
00010000:00000010:1.1:1788572325.536678:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4800.
00010000:00000010:1.1:1788572325.536686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99600.
00000080:00000010:2.1:1788572325.555658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5760.
00000080:00000010:2.1:1788572325.555687:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89535270eb40.
00010000:00000010:1.1:1788572325.562615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572325.562629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:1.1:1788572325.566631:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00000080:00000010:1.1:1788572325.570673:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519bdac0.
00010000:00000010:1.1:1788572325.583194:0:1046:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572325.583210:0:1046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:1.1:1788572325.591453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ba0.
00000080:00000010:1.1:1788572325.591481:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953519818c0.
00010000:00000010:1.1:1788572325.599685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074700.
00010000:00000010:1.1:1788572325.599692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:0.1:1788572325.607834:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5760.
00000080:00000010:0.1:1788572325.607874:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953544ddac0.
00010000:00000010:2.1:1788572325.700824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00010000:00000010:2.1:1788572325.700837:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56300.
00010000:00000010:2.1:1788572325.700841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00010000:00000010:2.1:1788572325.700845:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab400.
00010000:00000010:2.1:1788572325.700848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52400.
00000080:00000010:2.1:1788572325.707594:0:206949:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8953420fa100.
00000080:00000010:2.1:1788572325.707610:0:206949:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbea0.
00010000:00000010:3.1:1788572336.646433:0:207524:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572336.646444:0:207524:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:3.1:1788572336.679714:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:3.1:1788572336.708832:0:2365:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:3.1:1788572336.708840:0:2365:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00010000:00000010:0.1:1788572336.725016:0:207521:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074000.
00010000:00000010:0.1:1788572336.725091:0:207521:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:0.1:1788572336.780564:0:655:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b820.
00010000:00000010:2.1:1788572336.883663:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572336.883681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:2.1:1788572336.883686:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572336.888450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572337.228570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1c0.
00000080:00000010:2.1:1788572337.250467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895343a6ea00.
00000080:00000010:2.1:1788572337.250488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5280.
00010000:00000010:2.1:1788572337.277239:0:207383:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:2.1:1788572337.277250:0:207383:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98600.
00010000:00000010:2.1:1788572337.277254:0:207383:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab100.
00010000:00000010:2.1:1788572337.277258:0:207383:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52c00.
00010000:00000010:2.1:1788572337.277262:0:207383:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4d00.
00010000:00000010:2.1:1788572337.277265:0:207383:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c98e00.
00010000:00000010:2.1:1788572337.277268:0:207383:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4500.
00010000:00000010:2.1:1788572337.277271:0:207383:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895361c99c00.
00010000:00000010:1.1:1788572337.372795:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:1.1:1788572337.372808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:1.1:1788572337.403522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:0.1:1788572337.408200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00010000:00000010:2.1:1788572337.408554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:2.1:1788572337.408562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:2.1:1788572337.604448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572337.604465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572337.768240:0:207543:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb160.
00010000:00000010:3.1:1788572337.824001:0:207383:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:3.1:1788572337.833439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:0.1:1788572337.946465:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:0.1:1788572337.946472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:1.1:1788572337.976601:0:207550:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb160.
00010000:00000010:0.1:1788572338.046694:0:207549:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abc00.
00010000:00000010:0.1:1788572338.046701:0:207549:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00010000:00000010:2.1:1788572338.094859:0:207552:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00010000:00000010:3.1:1788572338.111419:0:207560:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:3.1:1788572338.111431:0:207560:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34d000.
00010000:00000010:2.1:1788572338.134407:0:207553:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:3.1:1788572345.029899:0:207585:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa51c0.
00010000:00000010:3.1:1788572345.029910:0:207585:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4300.
00010000:00000010:3.1:1788572345.029926:0:207585:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00010000:00000010:3.1:1788572345.029929:0:207585:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d34dc00.
00010000:00000010:3.1:1788572345.029933:0:207585:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074400.
00010000:00000010:3.1:1788572345.029936:0:207585:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00010000:00000010:2.1:1788572345.145682:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:2.1:1788572345.145695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572346.139476:0:207588:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf20.
00010000:00000010:3.1:1788572346.168434:0:207588:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572346.168449:0:207588:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:3.1:1788572346.241415:0:207594:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953726cee00.
00010000:00000010:3.1:1788572346.241427:0:207594:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00010000:00000010:3.1:1788572346.258422:0:207594:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:3.1:1788572346.258431:0:207594:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953726ce600.
00010000:00000010:2.1:1788572346.281016:0:207593:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572352.140723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b3a0.
00000800:00000200:1.1:1788572352.193229:0:2354:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace -24152 low water 2109228 conn ffff895342d4e000 blocked idle queued
00000800:00000200:1.1:1788572352.193244:0:2354:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace -24152 low water 2109228 conn ffff895342d4e000 blocked idle queued
00000800:00000200:1.1:1788572352.193256:0:2354:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace -24152 low water 2109228 conn ffff895342d4e000 blocked idle queued
00000800:00000200:1.1:1788572352.193268:0:2354:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace -24152 low water 2109228 conn ffff895342d4e000 blocked idle queued
00000800:00000200:1.1:1788572352.193280:0:2354:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace -24152 low water 2109228 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.201529:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1318084 low water 1438110 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.201546:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1318084 low water 1438110 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.202787:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1318084 low water 1438110 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.202829:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1318084 low water 1438110 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.202843:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1318084 low water 1438110 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.202857:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1318084 low water 1438110 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.202870:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1318084 low water 1438110 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.204381:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1318084 low water 1438110 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.204505:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1318084 low water 1438110 conn ffff895342d4e000 blocked idle queued
00000800:00000200:3.1:1788572352.204532:0:186212:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff895350788000 wspace 1445916 low water 1374194 conn ffff895342d4e000 blocked idle queued
00010000:00000010:2.1:1788572352.211528:0:1046:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab800.
00010000:00000010:2.1:1788572352.211538:0:1046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:2.1:1788572352.211542:0:1046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00010000:00000010:2.1:1788572352.211545:0:1046:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572352.211549:0:1046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:1.1:1788572352.659430:0:207629:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab000.
00010000:00000010:1.1:1788572352.659437:0:207629:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52400.
00000080:00000010:2.1:1788572354.034496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b040.
00010000:00000010:3.1:1788572354.117950:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074e00.
00010000:00000010:3.1:1788572354.117963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536cc7dc00.
00010000:00000010:3.1:1788572354.138543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00000080:00000010:0.1:1788572354.877708:0:207638:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b360.
00010000:00000010:0.1:1788572354.918163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00010000:00000010:0.1:1788572354.918175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:0.1:1788572354.918190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:1.1:1788572355.056804:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:1.1:1788572355.056818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:1.1:1788572355.177834:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb900.
00010000:00000010:3.1:1788572355.240918:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:3.1:1788572355.240934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:3.1:1788572355.243650:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00000080:00000010:2.1:1788572355.917526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b280.
00010000:00000010:0.1:1788572355.919604:0:207383:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:0.1:1788572355.919621:0:207383:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074600.
00010000:00000010:0.1:1788572355.919627:0:207383:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f9f000.
00010000:00000010:3.1:1788572356.015709:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572356.015716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372f9e400.
00000080:00000010:3.1:1788572356.065573:0:207653:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a20.
00010000:00000010:2.1:1788572356.093816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:2.1:1788572356.760722:0:207654:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb700.
00010000:00000010:0.1:1788572356.826959:0:207655:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:1.1:1788572356.915041:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00010000:00000010:3.1:1788572356.917644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:3.1:1788572356.917655:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572356.917660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00010000:00000010:3.1:1788572356.917662:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:3.1:1788572356.917664:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:2.1:1788572357.163454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572357.163471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00000080:00000010:2.1:1788572357.290463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5300.
00010000:00000010:1.1:1788572357.311675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895341518c00.
00010000:00000010:2.1:1788572358.179320:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074200.
00010000:00000010:2.1:1788572358.179329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00000080:00000010:3.1:1788572358.354447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5d80.
00000080:00000010:1.1:1788572358.416619:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb580.
00010000:00000010:2.1:1788572358.421417:0:207669:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:2.1:1788572358.421430:0:207669:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4400.
00010000:00000010:2.1:1788572358.421435:0:207669:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49c00.
00010000:00000010:2.1:1788572358.421439:0:207669:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572358.421442:0:207669:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:0.1:1788572358.537931:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412abb00.
00010000:00000010:0.1:1788572358.537938:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00010000:00000010:2.1:1788572358.562812:0:207674:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00000080:00000010:2.1:1788572358.617453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65bbe0.
00010000:00000010:1.1:1788572358.647462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947da00.
00010000:00000010:2.1:1788572363.593448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572363.593462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572363.922094:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5aa0.
00000080:00000010:1.1:1788572364.065510:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5620.
00010000:00000010:2.1:1788572364.065839:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49800.
00010000:00000010:2.1:1788572364.065847:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:2.1:1788572364.065851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89525947d400.
00010000:00000010:2.1:1788572364.065855:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572364.065858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:0.1:1788572364.296837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00010000:00000010:2.1:1788572364.303416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:2.1:1788572364.303432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:2.1:1788572364.343854:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572364.401613:0:207687:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00010000:00000010:3.1:1788572364.436450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572364.436461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:3.1:1788572364.436465:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:3.1:1788572364.436467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572364.601503:0:1377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b160.
00010000:00000010:3.1:1788572364.626920:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572364.626936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:3.1:1788572364.626940:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4c00.
00010000:00000010:3.1:1788572364.626945:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48a00.
00010000:00000010:2.1:1788572364.829565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572364.829582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:2.1:1788572364.886378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbea0.
00010000:00000010:3.1:1788572364.926497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab600.
00010000:00000010:3.1:1788572364.926519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00010000:00000010:3.1:1788572364.926523:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:3.1:1788572364.926526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:3.1:1788572365.087464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48000.
00000080:00000010:2.1:1788572365.175499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb940.
00010000:00000010:2.1:1788572365.183623:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572365.183639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00010000:00000010:2.1:1788572365.183642:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572365.183645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:2.1:1788572380.956924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00000080:00000010:1.1:1788572380.969456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7e0.
00010000:00000010:1.1:1788572380.969479:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab300.
00010000:00000010:1.1:1788572380.969484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:2.1:1788572381.089504:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572381.089520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:3.1:1788572381.144635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb60.
00010000:00000010:0.1:1788572381.179502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48c00.
00000080:00000010:2.1:1788572381.208635:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa54e0.
00010000:00000010:0.1:1788572381.234432:0:208344:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00000080:00000010:1.1:1788572381.261438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbdc0.
00010000:00000010:2.1:1788572381.788636:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953601f4200.
00010000:00000010:2.1:1788572381.788648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49a00.
00010000:00000010:2.1:1788572381.822431:0:208352:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:2.1:1788572381.822437:0:208352:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:2.1:1788572381.822441:0:208352:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00000080:00000010:3.1:1788572381.856473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb1a0.
00010000:00000010:2.1:1788572381.894733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953487ef600.
00000080:00000010:3.1:1788572381.922482:0:208355:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5420.
00010000:00000010:1.1:1788572381.992264:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074100.
00010000:00000010:1.1:1788572381.992280:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953487ee400.
00010000:00000010:1.1:1788572381.994673:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074a00.
00010000:00000010:2.1:1788572390.174159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:2.1:1788572390.174173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:2.1:1788572390.174178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572390.174180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:2.1:1788572390.174183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab500.
00010000:00000010:2.1:1788572390.174187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53a00.
00010000:00000010:3.1:1788572390.203283:0:208606:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:3.1:1788572390.203297:0:208606:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:3.1:1788572390.203302:0:208606:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56b00.
00010000:00000010:3.1:1788572390.203304:0:208606:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:3.1:1788572390.203307:0:208606:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572390.203309:0:208606:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572395.774484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbba0.
00010000:00000010:2.1:1788572395.795491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00010000:00000010:2.1:1788572395.824452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00010000:00000010:3.1:1788572395.923438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:1.1:1788572395.960553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb620.
00000080:00000010:1.1:1788572395.960613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5320.
00000080:00000010:1.1:1788572395.960618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b000.
00000080:00000010:1.1:1788572395.960623:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb200.
00000080:00000010:1.1:1788572395.960627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5a00.
00010000:00000010:1.1:1788572395.975612:0:208145:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00010000:00000010:1.1:1788572395.975625:0:208145:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:1.1:1788572395.975687:0:208145:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:1.1:1788572395.975693:0:208145:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953487ee200.
00010000:00000010:1.1:1788572395.975696:0:208145:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff8953412ab900.
00010000:00000010:1.1:1788572395.975699:0:208145:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00010000:00000010:1.1:1788572395.975701:0:208145:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:1.1:1788572395.975704:0:208145:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00010000:00000010:1.1:1788572395.975707:0:208145:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:1.1:1788572395.975709:0:208145:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:1.1:1788572395.975715:0:208145:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:1.1:1788572395.975718:0:208145:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:1.1:1788572395.975720:0:208145:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff89536d074b00.
00010000:00000010:1.1:1788572395.975723:0:208145:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48400.
00000080:00000010:2.1:1788572396.223746:0:208631:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ee0.
00000080:00000010:2.1:1788572396.223761:0:208631:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5c80.
00000080:00000010:2.1:1788572396.223765:0:208631:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5760.
00010000:00000010:2.1:1788572396.250961:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471500.
00010000:00000010:2.1:1788572396.250975:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656f0c00.
00010000:00000010:2.1:1788572396.250991:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471400.
00010000:00000010:2.1:1788572396.250995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656f1600.
00010000:00000010:2.1:1788572396.250997:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471c00.
00010000:00000010:2.1:1788572396.251000:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656f1200.
00010000:00000010:2.1:1788572396.251002:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471900.
00010000:00000010:2.1:1788572396.251005:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656f1800.
00000080:00000010:3.1:1788572396.436596:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb740.
00000080:00000010:3.1:1788572396.448545:0:208642:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb280.
00000080:00000010:3.1:1788572396.448563:0:208642:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3e0.
00010000:00000010:3.1:1788572396.451565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572396.451580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:3.1:1788572396.451584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0200.
00010000:00000010:3.1:1788572396.451588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344a93400.
00010000:00000010:3.1:1788572396.451590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56300.
00010000:00000010:3.1:1788572396.451594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:3.1:1788572396.451596:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:3.1:1788572396.451598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:2.1:1788572396.638000:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344a92c00.
00010000:00000010:2.1:1788572396.732462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572396.732474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00010000:00000010:2.1:1788572396.732477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:2.1:1788572396.748450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb00.
00000080:00000010:2.1:1788572396.752615:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346346300.
00010000:00000010:2.1:1788572396.768615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572396.768628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572396.784985:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00000080:00000010:2.1:1788572396.785017:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346342940.
00000080:00000010:2.1:1788572396.785025:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b5a0.
00010000:00000010:2.1:1788572396.800003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00010000:00000010:2.1:1788572396.800017:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0c00.
00010000:00000010:2.1:1788572396.800022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344a92000.
00010000:00000010:2.1:1788572396.800026:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572396.800029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00000080:00000010:2.1:1788572396.840421:0:208651:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff89534634c200.
00000080:00000010:2.1:1788572396.840435:0:208651:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb00.
00010000:00000010:1.1:1788572399.779853:0:208724:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a300.
00010000:00000010:1.1:1788572399.779870:0:208724:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52400.
00010000:00000010:1.1:1788572399.779874:0:208724:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:1.1:1788572399.779878:0:208724:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:2.1:1788572410.340794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00000080:00000010:2.1:1788572410.410443:0:209274:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbc60.
00010000:00000010:2.1:1788572410.410456:0:209274:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7af00.
00010000:00000010:2.1:1788572410.412619:0:209274:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00010000:00000010:2.1:1788572410.496083:0:209275:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a700.
00010000:00000010:2.1:1788572410.496101:0:209275:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:3.1:1788572410.531432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b940.
00010000:00000010:2.1:1788572410.571237:0:209276:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895344a92000.
00000080:00000010:3.1:1788572410.597806:0:209277:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56a0.
00010000:00000010:2.1:1788572410.623561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00000080:00000010:2.1:1788572410.634611:0:209278:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00010000:00000010:3.1:1788572411.043459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a600.
00010000:00000010:3.1:1788572411.043474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:2.1:1788572411.083814:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a400.
00010000:00000010:2.1:1788572411.083830:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572411.083834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52800.
00000080:00000010:2.1:1788572411.158176:0:209286:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b3e0.
00010000:00000010:2.1:1788572411.208430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572411.238466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cba80.
00010000:00000010:3.1:1788572411.310766:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572411.310779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:3.1:1788572411.334501:0:209291:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a600.
00010000:00000010:2.1:1788572418.392875:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a700.
00010000:00000010:2.1:1788572418.392887:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00010000:00000010:2.1:1788572418.392891:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471800.
00010000:00000010:2.1:1788572418.392894:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8953656f0000.
00010000:00000010:3.1:1788572418.416731:0:208825:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0400.
00010000:00000010:3.1:1788572418.416749:0:208825:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49200.
00010000:00000010:3.1:1788572418.416755:0:208825:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:3.1:1788572418.416759:0:208825:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00010000:00000010:0.1:1788572423.651517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:2.1:1788572423.827590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:3.1:1788572423.985184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:3.1:1788572424.048193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00000080:00000010:0.1:1788572424.185446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5ec0.
00000080:00000010:0.1:1788572424.185471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb980.
00000080:00000010:0.1:1788572424.185500:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b0c0.
00000080:00000010:0.1:1788572424.185505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b4a0.
00000080:00000010:0.1:1788572424.185508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb120.
00010000:00000010:2.1:1788572424.218432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572424.218447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:2.1:1788572424.218451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d085c00.
00010000:00000010:2.1:1788572424.218454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a800.
00010000:00000010:2.1:1788572424.218457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:2.1:1788572424.218460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48200.
00010000:00000010:2.1:1788572424.218463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:2.1:1788572424.218466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572424.218469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:2.1:1788572424.218471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:2.1:1788572424.218474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:2.1:1788572424.218477:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a200.
00010000:00000010:2.1:1788572424.218480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00010000:00000010:2.1:1788572424.218482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572424.218485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:3.1:1788572424.448433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5000.
00000080:00000010:3.1:1788572424.448552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb60.
00000080:00000010:3.1:1788572424.448557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c56e0.
00010000:00000010:3.1:1788572424.448564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:3.1:1788572424.448570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49400.
00010000:00000010:3.1:1788572424.448582:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471100.
00010000:00000010:3.1:1788572424.448586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d084600.
00010000:00000010:3.1:1788572424.448590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572424.448593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:3.1:1788572424.448597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:3.1:1788572424.448600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d084a00.
00000080:00000010:1.1:1788572424.727212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5d60.
00000080:00000010:1.1:1788572424.749529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65bec0.
00000080:00000010:1.1:1788572424.749548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b8a0.
00010000:00000010:1.1:1788572424.749555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ab00.
00010000:00000010:1.1:1788572424.749561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48a00.
00010000:00000010:1.1:1788572424.749566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:1.1:1788572424.749569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:1.1:1788572424.749572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a500.
00010000:00000010:1.1:1788572424.749586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:1.1:1788572424.749590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a900.
00010000:00000010:1.1:1788572424.749593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53200.
00010000:00000010:2.1:1788572424.980569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:2.1:1788572425.015451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572425.015468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01600.
00010000:00000010:2.1:1788572425.015471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:0.1:1788572425.019570:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346345ac0.
00000080:00000010:2.1:1788572425.033516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbae0.
00010000:00000010:0.1:1788572425.038507:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0b00.
00010000:00000010:0.1:1788572425.038522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49e00.
00000080:00000010:0.1:1788572425.046733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa54a0.
00000080:00000010:0.1:1788572425.046761:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895346344a40.
00000080:00000010:0.1:1788572425.055505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6a0.
00010000:00000010:2.1:1788572425.069436:0:209585:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49e00.
00010000:00000010:2.1:1788572425.069445:0:209585:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572425.069449:0:209585:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:2.1:1788572425.069453:0:209585:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0900.
00010000:00000010:2.1:1788572425.069456:0:209585:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49000.
00000080:00000010:2.1:1788572425.095509:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff895351a75280.
00000080:00000010:2.1:1788572425.095525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5540.
00010000:00000010:3.1:1788572425.334568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56300.
00010000:00000010:3.1:1788572425.334595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:2.1:1788572425.342545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572425.342558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:3.1:1788572436.133720:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ac00.
00010000:00000010:3.1:1788572436.133735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00010000:00000010:3.1:1788572436.162772:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ae00.
00010000:00000010:3.1:1788572436.162784:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ac00.
00010000:00000010:3.1:1788572436.162788:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52e00.
00000080:00000010:2.1:1788572436.227065:0:210195:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65ba40.
00010000:00000010:0.1:1788572436.413358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471800.
00010000:00000010:0.1:1788572436.413409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d085a00.
00010000:00000010:0.1:1788572436.443440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0300.
00010000:00000010:1.1:1788572436.456040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49600.
00000080:00000010:2.1:1788572436.687545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5520.
00010000:00000010:2.1:1788572436.913748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:0.1:1788572436.955017:0:210208:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f20.
00010000:00000010:2.1:1788572437.195858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572437.229741:0:210213:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbd80.
00010000:00000010:2.1:1788572450.372615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49400.
00010000:00000010:2.1:1788572450.372632:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471f00.
00010000:00000010:2.1:1788572450.372636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00010000:00000010:2.1:1788572450.387442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572450.400638:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:2.1:1788572450.400653:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:1.1:1788572450.427587:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:1.1:1788572450.427601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:1.1:1788572450.453608:0:210842:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ba0.
00010000:00000010:3.1:1788572450.482456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0200.
00010000:00000010:3.1:1788572450.482481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49400.
00010000:00000010:3.1:1788572450.501547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ab00.
00010000:00000010:1.1:1788572450.627623:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471400.
00010000:00000010:1.1:1788572450.627635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00010000:00000010:1.1:1788572450.654659:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471500.
00010000:00000010:0.1:1788572450.657595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00000080:00000010:2.1:1788572450.701963:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65bc80.
00010000:00000010:1.1:1788572450.765279:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00000080:00000010:2.1:1788572450.787438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b320.
00010000:00000010:3.1:1788572450.868535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49600.
00000080:00000010:3.1:1788572450.888019:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5ca0.
00010000:00000010:1.1:1788572459.526747:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7ab00.
00010000:00000010:1.1:1788572459.526762:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53a00.
00010000:00000010:2.1:1788572459.542436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471200.
00010000:00000010:0.1:1788572459.801876:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:0.1:1788572459.801884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:2.1:1788572459.827203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:2.1:1788572459.827230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:2.1:1788572459.827235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572459.827237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:3.1:1788572459.901720:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56a00.
00010000:00000010:3.1:1788572459.901735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00010000:00000010:2.1:1788572459.912463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28800.
00010000:00000010:3.1:1788572459.936644:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7af00.
00010000:00000010:3.1:1788572459.982495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00000080:00000010:0.1:1788572460.175529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5fe0.
00000080:00000010:1.1:1788572460.216443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c59a0.
00010000:00000010:3.1:1788572460.244434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471000.
00010000:00000010:3.1:1788572460.244448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:1.1:1788572460.276846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49800.
00000080:00000010:1.1:1788572460.365430:0:211438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa57a0.
00010000:00000010:0.1:1788572460.388997:0:2355:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471700.
00010000:00000010:0.1:1788572460.389008:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00010000:00000010:2.1:1788572460.393740:0:1106:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0100.
00010000:00000010:0.1:1788572460.413659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49e00.
00010000:00000010:0.1:1788572460.420320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00010000:00000010:3.1:1788572460.450501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:2.1:1788572460.667138:0:211439:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5620.
00000080:00000010:3.1:1788572460.727375:0:1377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb920.
00010000:00000010:2.1:1788572460.754517:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471d00.
00010000:00000010:2.1:1788572460.754530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49e00.
00010000:00000010:2.1:1788572460.788159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49e00.
00000080:00000010:2.1:1788572460.832752:0:211276:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5b00.
00010000:00000010:0.1:1788572460.876563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:0.1:1788572460.876576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:0.1:1788572460.889435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48c00.
00010000:00000010:2.1:1788572460.889501:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0e00.
00010000:00000010:2.1:1788572460.889507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:2.1:1788572460.912554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00000080:00000010:2.1:1788572461.141463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5720.
00000080:00000010:2.1:1788572461.201988:0:211444:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65bb00.
00010000:00000010:3.1:1788572461.227715:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471800.
00010000:00000010:3.1:1788572461.227729:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49c00.
00010000:00000010:0.1:1788572461.253952:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:2.1:1788572461.295562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5e00.
00010000:00000010:1.1:1788572461.311806:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7af00.
00010000:00000010:1.1:1788572461.311819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00010000:00000010:3.1:1788572461.323853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00010000:00000010:0.1:1788572461.324024:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471600.
00010000:00000010:0.1:1788572461.333513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00010000:00000010:2.1:1788572461.367571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48000.
00000080:00000010:2.1:1788572461.606271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f80.
00000080:00000010:1.1:1788572461.707919:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5480.
00010000:00000010:3.1:1788572461.751496:0:211450:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0800.
00010000:00000010:3.1:1788572461.751518:0:211450:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00010000:00000010:0.1:1788572461.789562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00000080:00000010:2.1:1788572461.864792:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65bfc0.
00010000:00000010:3.1:1788572461.888486:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a400.
00010000:00000010:3.1:1788572461.888506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00010000:00000010:3.1:1788572461.902313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7af00.
00010000:00000010:2.1:1788572461.905449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:3.1:1788572461.924085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48000.
00010000:00000010:3.1:1788572461.963455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00000080:00000010:2.1:1788572462.256807:0:211452:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b540.
00000080:00000010:3.1:1788572462.366548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbf60.
00010000:00000010:0.1:1788572462.404838:0:211454:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:0.1:1788572462.404854:0:211454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:3.1:1788572462.452502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48000.
00000080:00000010:2.1:1788572462.536661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55a0.
00010000:00000010:2.1:1788572462.551816:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:2.1:1788572462.551828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:1.1:1788572462.555723:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572462.561091:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:3.1:1788572462.561106:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:3.1:1788572462.622094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:0.1:1788572462.813652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb3a0.
00000080:00000010:2.1:1788572462.963311:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa53a0.
00010000:00000010:3.1:1788572462.984664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471e00.
00010000:00000010:3.1:1788572462.984678:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00010000:00000010:3.1:1788572463.030831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00000080:00000010:2.1:1788572463.117536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa59c0.
00010000:00000010:1.1:1788572463.147588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7af00.
00010000:00000010:1.1:1788572463.147595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52400.
00010000:00000010:3.1:1788572463.154194:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471800.
00010000:00000010:0.1:1788572463.154600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00010000:00000010:1.1:1788572463.154734:0:2354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48800.
00010000:00000010:2.1:1788572463.166847:0:2364:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00000080:00000010:2.1:1788572463.315976:0:1377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5560.
00000080:00000010:0.1:1788572463.390440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbe00.
00010000:00000010:3.1:1788572463.456429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56400.
00010000:00000010:3.1:1788572463.456440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:3.1:1788572463.484780:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00000080:00000010:2.1:1788572463.530542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5f60.
00010000:00000010:2.1:1788572463.588894:0:211465:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471100.
00010000:00000010:2.1:1788572463.588905:0:211465:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00010000:00000010:2.1:1788572463.601598:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:3.1:1788572463.604974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00010000:00000010:2.1:1788572463.613242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:3.1:1788572463.629512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00000080:00000010:2.1:1788572463.734657:0:211466:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb780.
00000080:00000010:2.1:1788572463.814645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb320.
00010000:00000010:0.1:1788572463.852499:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:0.1:1788572463.852514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49200.
00010000:00000010:3.1:1788572463.875101:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:2.1:1788572463.919256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbac0.
00010000:00000010:1.1:1788572463.983451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:1.1:1788572463.983468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:3.1:1788572463.987437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28e00.
00010000:00000010:2.1:1788572463.987489:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471d00.
00010000:00000010:2.1:1788572463.993429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:3.1:1788572464.035488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:2.1:1788572464.293299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c54a0.
00000080:00000010:2.1:1788572464.434775:0:211276:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa50c0.
00010000:00000010:3.1:1788572464.503839:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:3.1:1788572464.503856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:1.1:1788572464.596465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:2.1:1788572464.671529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65bb60.
00010000:00000010:2.1:1788572464.676473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0d00.
00010000:00000010:2.1:1788572464.676484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48a00.
00010000:00000010:2.1:1788572464.693082:0:2364:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572464.696568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:2.1:1788572464.713579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00010000:00000010:3.1:1788572464.750633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52c00.
00000080:00000010:0.1:1788572465.120364:0:211474:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb780.
00000080:00000010:2.1:1788572465.260088:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b820.
00010000:00000010:1.1:1788572465.268573:0:211476:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a400.
00010000:00000010:1.1:1788572465.268590:0:211476:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:3.1:1788572465.282455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00000080:00000010:3.1:1788572465.304724:0:211477:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5820.
00010000:00000010:3.1:1788572465.330947:0:211477:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0b00.
00010000:00000010:3.1:1788572465.330955:0:211477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48000.
00010000:00000010:0.1:1788572465.335450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00010000:00000010:3.1:1788572465.342613:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a100.
00010000:00000010:3.1:1788572465.342622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52c00.
00010000:00000010:3.1:1788572465.359443:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:2.1:1788572465.566678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b3e0.
00000080:00000010:1.1:1788572465.604515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5de0.
00010000:00000010:0.1:1788572465.651441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471300.
00010000:00000010:0.1:1788572465.651448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:1.1:1788572465.660695:0:2363:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52c00.
00000080:00000010:3.1:1788572465.676697:0:211481:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b780.
00010000:00000010:3.1:1788572465.710129:0:208825:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471000.
00010000:00000010:3.1:1788572465.710140:0:208825:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00010000:00000010:3.1:1788572465.723732:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a600.
00010000:00000010:1.1:1788572465.723951:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52c00.
00010000:00000010:0.1:1788572465.733604:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00010000:00000010:2.1:1788572465.753478:0:2364:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53000.
00000080:00000010:1.1:1788572465.851618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b1c0.
00000080:00000010:2.1:1788572465.928730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa57c0.
00010000:00000010:2.1:1788572465.977652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a100.
00010000:00000010:2.1:1788572465.977667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00010000:00000010:1.1:1788572466.020908:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49000.
00000080:00000010:2.1:1788572466.129902:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5240.
00010000:00000010:1.1:1788572466.155799:0:211486:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0600.
00010000:00000010:1.1:1788572466.155815:0:211486:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49600.
00010000:00000010:1.1:1788572466.195716:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471a00.
00010000:00000010:3.1:1788572466.212174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00010000:00000010:3.1:1788572466.223227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49800.
00010000:00000010:3.1:1788572466.225525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48800.
00000080:00000010:2.1:1788572466.484527:0:211487:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c55c0.
00000080:00000010:3.1:1788572466.560438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbfc0.
00010000:00000010:2.1:1788572466.584515:0:211489:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0700.
00010000:00000010:2.1:1788572466.584532:0:211489:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00010000:00000010:1.1:1788572466.638962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01800.
00000080:00000010:0.1:1788572466.680714:0:211490:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00010000:00000010:0.1:1788572466.708515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:0.1:1788572466.708523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01000.
00010000:00000010:2.1:1788572466.708577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0400.
00010000:00000010:3.1:1788572466.712507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49800.
00010000:00000010:2.1:1788572466.717731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:3.1:1788572466.752063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49a00.
00000080:00000010:2.1:1788572466.910250:0:211491:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5260.
00000080:00000010:3.1:1788572466.956565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb400.
00010000:00000010:2.1:1788572467.016221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0900.
00010000:00000010:2.1:1788572467.016239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:3.1:1788572467.028571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49000.
00000080:00000010:2.1:1788572467.080456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5de0.
00010000:00000010:2.1:1788572467.088308:0:211494:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:2.1:1788572467.088320:0:211494:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00010000:00000010:3.1:1788572467.110455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01200.
00010000:00000010:2.1:1788572467.110721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:2.1:1788572467.117229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00010000:00000010:2.1:1788572467.153478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:1.1:1788572467.337574:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb4e0.
00000080:00000010:3.1:1788572467.400437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb560.
00010000:00000010:0.1:1788572467.450051:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56000.
00010000:00000010:0.1:1788572467.450202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00000.
00010000:00000010:3.1:1788572467.495437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:2.1:1788572467.570133:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5fe0.
00010000:00000010:1.1:1788572467.641565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56700.
00010000:00000010:1.1:1788572467.641577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:0.1:1788572467.673509:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471a00.
00010000:00000010:3.1:1788572467.687133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00010000:00000010:2.1:1788572467.691796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49400.
00010000:00000010:3.1:1788572467.692555:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00000080:00000010:2.1:1788572467.800549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5cc0.
00000080:00000010:2.1:1788572467.847734:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b7c0.
00010000:00000010:2.1:1788572467.886495:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0600.
00010000:00000010:2.1:1788572467.886509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:3.1:1788572467.937441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48000.
00000080:00000010:2.1:1788572468.026428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa54c0.
00010000:00000010:1.1:1788572468.039945:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0d00.
00010000:00000010:1.1:1788572468.039960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48c00.
00010000:00000010:1.1:1788572468.060078:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471200.
00010000:00000010:2.1:1788572468.062431:0:2365:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29c00.
00010000:00000010:0.1:1788572468.071981:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53800.
00010000:00000010:2.1:1788572468.109235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49e00.
00000080:00000010:2.1:1788572468.258669:0:211504:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a80.
00000080:00000010:2.1:1788572468.295499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbee0.
00010000:00000010:3.1:1788572468.326779:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0800.
00010000:00000010:3.1:1788572468.326790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:3.1:1788572468.349105:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00000080:00000010:3.1:1788572468.384727:0:211507:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5280.
00010000:00000010:3.1:1788572468.398846:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56f00.
00010000:00000010:3.1:1788572468.398929:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:3.1:1788572468.420702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a600.
00010000:00000010:2.1:1788572468.426055:0:2365:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53e00.
00010000:00000010:1.1:1788572468.441840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:2.1:1788572468.462428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00000080:00000010:3.1:1788572468.592197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65bc60.
00000080:00000010:2.1:1788572468.641917:0:47:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbda0.
00010000:00000010:3.1:1788572468.667757:0:211510:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471200.
00010000:00000010:3.1:1788572468.667770:0:211510:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00010000:00000010:0.1:1788572468.704523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52000.
00000080:00000010:1.1:1788572468.740764:0:211276:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65bea0.
00010000:00000010:2.1:1788572468.779786:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:2.1:1788572468.779801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:2.1:1788572468.783773:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:0.1:1788572468.784454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:2.1:1788572468.802958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:2.1:1788572468.812054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:1.1:1788572468.963498:0:211512:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb6c0.
00000080:00000010:2.1:1788572469.013998:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb7a0.
00010000:00000010:3.1:1788572469.030521:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56d00.
00010000:00000010:3.1:1788572469.030538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:2.1:1788572469.056739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49400.
00000080:00000010:1.1:1788572469.075547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5e20.
00010000:00000010:1.1:1788572469.134452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471500.
00010000:00000010:1.1:1788572469.134469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00010000:00000010:2.1:1788572469.172632:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471400.
00010000:00000010:0.1:1788572469.181453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29e00.
00010000:00000010:2.1:1788572469.197988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:3.1:1788572469.215134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48800.
00000080:00000010:2.1:1788572469.434642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5aa0.
00000080:00000010:3.1:1788572469.481267:0:211519:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5900.
00010000:00000010:3.1:1788572469.511539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0e00.
00010000:00000010:3.1:1788572469.511557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:0.1:1788572469.554415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00000080:00000010:2.1:1788572469.646789:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c53c0.
00010000:00000010:0.1:1788572469.693134:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0000.
00010000:00000010:0.1:1788572469.693146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49a00.
00010000:00000010:0.1:1788572469.747516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0c00.
00010000:00000010:3.1:1788572469.747944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49200.
00010000:00000010:0.1:1788572469.776205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:3.1:1788572469.820690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00000080:00000010:2.1:1788572470.049730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5be0.
00000080:00000010:0.1:1788572470.088760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5820.
00010000:00000010:2.1:1788572470.132746:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:2.1:1788572470.132760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:3.1:1788572470.162756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00000080:00000010:0.1:1788572470.200516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5380.
00010000:00000010:1.1:1788572470.221860:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471500.
00010000:00000010:1.1:1788572470.221867:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00010000:00000010:2.1:1788572470.227536:0:179193:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471f00.
00010000:00000010:2.1:1788572470.234446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49600.
00010000:00000010:2.1:1788572470.234459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29600.
00010000:00000010:2.1:1788572470.268775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48800.
00000080:00000010:0.1:1788572470.401736:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5840.
00000080:00000010:0.1:1788572470.455678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5260.
00010000:00000010:3.1:1788572470.495427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0c00.
00010000:00000010:3.1:1788572470.495451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:1.1:1788572470.508434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29000.
00000080:00000010:2.1:1788572470.535431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c57a0.
00010000:00000010:1.1:1788572470.543918:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471500.
00010000:00000010:1.1:1788572470.543928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28c00.
00010000:00000010:3.1:1788572470.550695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471900.
00010000:00000010:2.1:1788572470.555160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28200.
00010000:00000010:3.1:1788572470.558711:0:211529:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29200.
00010000:00000010:2.1:1788572470.583427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00000080:00000010:2.1:1788572470.682443:0:211530:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5500.
00000080:00000010:0.1:1788572470.789677:0:1377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb500.
00010000:00000010:3.1:1788572470.841532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572470.841548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49c00.
00010000:00000010:1.1:1788572470.868526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:2.1:1788572470.907717:0:211533:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5a00.
00010000:00000010:3.1:1788572470.964337:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:3.1:1788572470.964347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00010000:00000010:3.1:1788572470.972769:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a600.
00010000:00000010:2.1:1788572470.972843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52400.
00010000:00000010:3.1:1788572470.985651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:2.1:1788572470.990430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:1.1:1788572471.111555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b880.
00000080:00000010:2.1:1788572471.176449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b360.
00010000:00000010:3.1:1788572471.232581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:3.1:1788572471.232599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:2.1:1788572471.284443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48800.
00000080:00000010:3.1:1788572471.358442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5fa0.
00010000:00000010:2.1:1788572471.378039:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0f00.
00010000:00000010:2.1:1788572471.378055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48000.
00010000:00000010:2.1:1788572471.405497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:0.1:1788572471.409502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:0.1:1788572471.461457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53400.
00010000:00000010:3.1:1788572471.500762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48c00.
00000080:00000010:2.1:1788572471.768462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb5a0.
00000080:00000010:3.1:1788572471.838418:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89534b65b5e0.
00010000:00000010:3.1:1788572471.877347:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0c00.
00010000:00000010:3.1:1788572471.877362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01e00.
00010000:00000010:0.1:1788572471.909721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29400.
00000080:00000010:3.1:1788572471.943738:0:211542:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5000.
00010000:00000010:2.1:1788572471.967071:0:1046:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a600.
00010000:00000010:2.1:1788572471.967084:0:1046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52a00.
00010000:00000010:2.1:1788572471.981762:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:1.1:1788572471.982152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00400.
00010000:00000010:2.1:1788572471.985443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53600.
00010000:00000010:2.1:1788572471.991305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52200.
00000080:00000010:0.1:1788572472.114462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb560.
00000080:00000010:1.1:1788572472.149551:0:211544:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c50c0.
00010000:00000010:2.1:1788572472.189436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a800.
00010000:00000010:2.1:1788572472.189456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00010000:00000010:1.1:1788572472.222834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49800.
00000080:00000010:2.1:1788572472.232560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa55c0.
00010000:00000010:0.1:1788572472.246753:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471500.
00010000:00000010:0.1:1788572472.246766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28a00.
00010000:00000010:0.1:1788572472.264416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0e00.
00010000:00000010:2.1:1788572472.266474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48800.
00010000:00000010:0.1:1788572472.275962:0:2355:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28600.
00010000:00000010:3.1:1788572472.290796:0:211546:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01c00.
00000080:00000010:2.1:1788572472.373632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa56e0.
00000080:00000010:1.1:1788572472.429590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5740.
00010000:00000010:2.1:1788572472.454195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:2.1:1788572472.454205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53a00.
00010000:00000010:1.1:1788572472.475533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00000080:00000010:1.1:1788572472.500545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cbb80.
00010000:00000010:2.1:1788572484.718649:0:212265:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0e00.
00010000:00000010:2.1:1788572484.718664:0:212265:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49600.
00010000:00000010:2.1:1788572484.738075:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:3.1:1788572484.743898:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56900.
00010000:00000010:3.1:1788572484.743923:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:2.1:1788572484.756470:0:212265:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:2.1:1788572484.756476:0:212265:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00000080:00000010:2.1:1788572484.772871:0:212267:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb8a0.
00010000:00000010:3.1:1788572493.759769:0:212680:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:3.1:1788572493.759785:0:212680:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00600.
00010000:00000010:3.1:1788572493.759830:0:212680:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56200.
00010000:00000010:3.1:1788572493.759834:0:212680:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00010000:00000010:0.1:1788572518.908497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:0.1:1788572518.908515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00200.
00010000:00000010:0.1:1788572518.916423:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0200.
00010000:00000010:1.1:1788572518.931409:0:1030:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0300.
00010000:00000010:1.1:1788572518.931416:0:1030:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49600.
00010000:00000010:2.1:1788572518.939965:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0c00.
00010000:00000010:2.1:1788572518.939972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49a00.
00000080:00000010:2.1:1788572518.978552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff895345fa5c80.
00010000:00000010:0.1:1788572527.510370:0:186668:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0200.
00010000:00000010:0.1:1788572527.510598:0:186668:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49e00.
00010000:00000010:0.1:1788572527.510603:0:186668:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0800.
00010000:00000010:0.1:1788572527.510607:0:186668:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00010000:00000010:3.1:1788572568.204454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471f00.
00010000:00000010:3.1:1788572568.204473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f28400.
00010000:00000010:2.1:1788572568.235285:0:186668:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7af00.
00010000:00000010:0.1:1788572568.245449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56e00.
00010000:00000010:0.1:1788572568.245520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d52600.
00010000:00000010:3.1:1788572568.336095:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56c00.
00010000:00000010:3.1:1788572568.336116:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00a00.
00000080:00000010:3.1:1788572568.342487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb340.
00010000:00000010:1.1:1788572568.418428:0:215619:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00000080:00000010:1.1:1788572568.418435:0:215619:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5500.
00010000:00000010:1.1:1788572577.979143:0:216036:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56500.
00010000:00000010:1.1:1788572577.979155:0:216036:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01a00.
00010000:00000010:0.1:1788572593.044860:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0c00.
00010000:00000010:0.1:1788572593.044874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49200.
00010000:00000010:0.1:1788572593.059989:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0000.
00010000:00000010:0.1:1788572593.059999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895380f29800.
00010000:00000010:3.1:1788572593.064724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a48e00.
00000800:00000200:3.1:1788572633.493617:0:2353:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1
00000800:00000200:3.1:1788572633.493620:0:2353:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:3.1:1788572633.493622:0:2353:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1
00000800:00000200:0.1:1788572635.542249:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572638.615165:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572638.615434:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572638.615629:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572643.734074:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572643.734470:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572643.734641:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572648.854306:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572648.854310:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 4, 1 secs later)
00000800:00000200:3.1:1788572648.855074:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572648.855077:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 4, 1 secs later)
00000800:00000200:3.1:1788572648.855447:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572648.855451:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 4, 1 secs later)
00000800:00000200:3.1:1788572653.974655:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572653.975426:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572653.975597:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572659.094070:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572659.094077:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 4 secs later)
00000800:00000200:0.1:1788572659.094607:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572659.094610:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 4 secs later)
00000800:00000200:0.1:1788572659.094897:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572659.094898:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 4 secs later)
00000800:00000200:0.1:1788572664.214524:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572664.214778:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572664.214937:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572669.334474:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572669.334486:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:0.1:1788572669.334998:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572669.335001:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:0.1:1788572669.335308:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572669.335311:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:0.1:1788572674.454527:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572674.454532:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:0.1:1788572674.454931:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572674.454934:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:0.1:1788572674.455253:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572674.455257:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:0.1:1788572679.574327:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572679.574336:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:0.1:1788572679.574954:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572679.574957:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:0.1:1788572679.575305:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572679.575308:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:0.1:1788572684.693925:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572684.694169:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572684.694369:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572689.814086:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572689.814092:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 27 secs later)
00000800:00000200:0.1:1788572689.814622:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572689.814626:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 27 secs later)
00000800:00000200:0.1:1788572689.814945:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572689.814949:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 27 secs later)
00000800:00000200:0.1:1788572694.934091:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572694.934099:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 22 secs later)
00000800:00000200:0.1:1788572694.934727:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572694.934730:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 22 secs later)
00000800:00000200:0.1:1788572694.935080:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572694.935082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 22 secs later)
00000800:00000200:3.1:1788572700.054305:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572700.054308:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 17 secs later)
00000800:00000200:3.1:1788572700.054898:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572700.054901:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 17 secs later)
00000800:00000200:3.1:1788572700.055224:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572700.055226:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 17 secs later)
00000800:00000200:3.1:1788572705.174178:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572705.174184:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 12 secs later)
00000800:00000200:3.1:1788572705.174791:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572705.174794:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 12 secs later)
00000800:00000200:3.1:1788572705.175056:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572705.175058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 12 secs later)
00000800:00000200:3.1:1788572710.294064:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572710.294070:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 7 secs later)
00000800:00000200:3.1:1788572710.294528:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572710.294531:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 7 secs later)
00000800:00000200:3.1:1788572710.294838:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572710.294840:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 7 secs later)
00000800:00000200:3.1:1788572715.414151:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572715.414156:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 2 secs later)
00000800:00000200:3.1:1788572715.414789:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572715.414792:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 2 secs later)
00000800:00000200:3.1:1788572715.415093:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572715.415095:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 2 secs later)
00000800:00000200:3.1:1788572720.534095:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572720.534433:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572720.534707:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572725.654209:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572725.654213:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1788572725.654679:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572725.654681:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1788572725.655013:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572725.655016:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1788572730.774250:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572730.774257:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 51 secs later)
00000800:00000200:3.1:1788572730.775236:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572730.775240:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 51 secs later)
00000800:00000200:3.1:1788572730.775596:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572730.775598:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 51 secs later)
00000800:00000200:0.1:1788572735.894202:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572735.894211:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 46 secs later)
00000800:00000200:0.1:1788572735.894880:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572735.894884:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 46 secs later)
00000800:00000200:0.1:1788572735.895231:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572735.895235:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 46 secs later)
00000100:00000010:1.0:1788572736.853492:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000020:00000001:1.0:1788572738.325569:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572738.325572:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572738.325575:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572739.349524:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572739.349529:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572739.349533:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788572741.013490:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572741.013499:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788572741.013501:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572741.013504:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572741.013508:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572741.013514:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 9672
00000400:00000200:1.0:1788572741.013522:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572741.013527:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572741.013529:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572741.013532:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013534:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572741.013536:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013538:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572741.013541:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572741.013543:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788572741.013545:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572741.013547:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788572741.013549:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788572741.013551:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788572741.013553:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013555:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572741.013557:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013558:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572741.013561:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788572741.013563:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013566:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572741.013569:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572741.013571:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572741.013574:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572741.013575:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788572741.013578:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572741.013583:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788572741.013586:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788572741.013589:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572741.013591:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572741.013593:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572741.013595:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572741.013598:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572741.013605:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788572741.013610:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572741.013618:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393006848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572741.013626:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572741.013629:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572741.013632:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393006848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572741.013654:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572741.013656:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572741.013659:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788572741.013660:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572741.013662:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572741.013665:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572741.013667:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9672
00000400:00000200:1.0:1788572741.013670:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572741.013672:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572741.013674:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572741.013677:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013679:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572741.013682:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013684:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572741.013687:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572741.013688:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788572741.013690:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572741.013692:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788572741.013695:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788572741.013710:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788572741.013714:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013717:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572741.013719:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013721:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572741.013725:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788572741.013731:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013735:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572741.013738:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572741.013743:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572741.013747:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572741.013752:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572741.013757:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788572741.013763:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572741.013765:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572741.013770:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788572741.013774:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788572741.013776:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572741.013779:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572741.013782:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572741.013792:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572741.013796:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572741.013801:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788572741.013807:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572741.013814:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393006976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572741.013825:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572741.013829:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572741.013832:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393006976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572741.013845:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572741.013848:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572741.013853:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788572741.013857:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572741.013859:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572741.013861:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572741.013866:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9672
00000400:00000200:1.0:1788572741.013870:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572741.013872:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572741.013874:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572741.013879:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013881:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572741.013882:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013885:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572741.013892:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572741.013893:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788572741.013896:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572741.013898:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788572741.013899:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788572741.013901:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788572741.013902:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013904:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572741.013905:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013907:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572741.013908:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788572741.013910:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572741.013912:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572741.013913:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572741.013915:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572741.013917:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572741.013921:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572741.013923:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788572741.013926:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572741.013928:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572741.013933:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788572741.013937:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788572741.013942:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572741.013944:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572741.013947:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572741.013950:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572741.013952:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572741.013956:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788572741.013959:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572741.013965:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393007104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572741.013970:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572741.013971:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572741.013973:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393007104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572741.013978:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572741.013980:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788572741.013986:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788572741.013990:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (9682)
00000800:00000200:3.1:1788572741.014106:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572741.014111:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 41 secs later)
00000800:00000200:3.1:1788572741.014505:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572741.014507:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 41 secs later)
00000800:00000200:3.1:1788572741.014720:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572741.014721:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 41 secs later)
00000100:00000040:1.0:1788572741.973458:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304345529, at 4304345536
00000100:00000010:1.0:1788572741.973474:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000020:00000001:1.0:1788572742.421677:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572742.421683:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572742.421687:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572743.445574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572743.445579:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572743.445582:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1788572746.133930:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572746.133934:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 36 secs later)
00000800:00000200:3.1:1788572746.134272:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572746.134274:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 36 secs later)
00000800:00000200:3.1:1788572746.134492:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572746.134494:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 36 secs later)
00000020:00000001:1.0:1788572746.517631:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572746.517634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572746.517637:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572747.093510:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304350649, at 4304350656
00000100:00000010:1.0:1788572747.093526:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000020:00000001:1.0:1788572747.541607:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572747.541610:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572747.541613:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572750.613601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572750.613604:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572750.613607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788572751.253513:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572751.253524:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788572751.253526:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572751.253530:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572751.253549:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572751.253556:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 9682
00000400:00000200:1.0:1788572751.253565:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572751.253572:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572751.253575:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572751.253579:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253582:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572751.253585:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253588:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572751.253593:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572751.253595:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788572751.253597:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572751.253600:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788572751.253603:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788572751.253605:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788572751.253609:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253611:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572751.253613:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253615:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572751.253619:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788572751.253622:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253624:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572751.253627:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572751.253630:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572751.253632:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572751.253634:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788572751.253635:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572751.253643:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788572751.253646:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788572751.253648:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572751.253650:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572751.253652:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572751.253654:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572751.253657:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572751.253664:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788572751.253669:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572751.253676:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393007616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572751.253683:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572751.253687:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572751.253690:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393007616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572751.253707:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572751.253711:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572751.253715:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788572751.253716:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572751.253718:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572751.253721:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572751.253724:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9682
00000400:00000200:1.0:1788572751.253728:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572751.253732:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572751.253734:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572751.253737:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253740:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572751.253742:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253744:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572751.253746:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572751.253747:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788572751.253749:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572751.253751:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788572751.253754:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788572751.253757:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788572751.253760:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253762:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572751.253764:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253766:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572751.253768:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788572751.253771:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253774:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572751.253776:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572751.253781:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572751.253783:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572751.253785:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572751.253789:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788572751.253792:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572751.253794:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572751.253797:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788572751.253799:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788572751.253802:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572751.253803:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572751.253805:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572751.253807:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572751.253808:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572751.253812:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788572751.253816:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572751.253821:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393007744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572751.253829:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572751.253831:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572751.253835:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393007744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572751.253844:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572751.253847:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572751.253850:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788572751.253852:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572751.253854:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572751.253856:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572751.253858:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9682
00000400:00000200:1.0:1788572751.253862:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572751.253865:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572751.253867:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572751.253880:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253882:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572751.253884:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253886:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572751.253888:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572751.253890:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788572751.253892:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572751.253894:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788572751.253896:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788572751.253898:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788572751.253901:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253903:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572751.253905:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253908:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572751.253910:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788572751.253913:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572751.253915:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572751.253917:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572751.253920:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572751.253922:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572751.253923:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572751.253925:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788572751.253928:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572751.253930:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572751.253933:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788572751.253935:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:3.1:1788572751.253937:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1788572751.253938:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572751.253939:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572751.253940:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 31 secs later)
00000100:00000001:1.0:1788572751.253941:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572751.253943:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572751.253944:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572751.253948:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788572751.253951:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572751.253955:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393007872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572751.253963:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572751.253965:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572751.253969:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393007872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572751.253976:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572751.253979:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788572751.253983:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788572751.253988:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (9692)
00000800:00000200:3.1:1788572751.254256:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572751.254258:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 31 secs later)
00000800:00000200:3.1:1788572751.254460:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572751.254462:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 31 secs later)
00000020:00000001:1.0:1788572751.637583:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572751.637587:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572751.637589:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572752.213497:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304355769, at 4304355776
00000100:00000010:1.0:1788572752.213514:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000800:00000200:1.0:1788572754.325797:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788572754.325802:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788572754.325814:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788572754.325819:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788572754.325821:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788572754.709595:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572754.709600:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572754.709603:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572755.733689:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572755.733694:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572755.733697:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572756.374070:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572756.374076:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 26 secs later)
00000800:00000200:0.1:1788572756.374429:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572756.374431:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 26 secs later)
00000800:00000200:0.1:1788572756.374738:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572756.374740:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 26 secs later)
00000100:00000040:1.0:1788572757.333501:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304360889, at 4304360896
00000100:00000010:1.0:1788572757.333521:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7c00.
00000020:00000001:1.0:1788572758.805598:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572758.805602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572758.805605:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572759.829605:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572759.829609:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572759.829612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572761.494046:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572761.494050:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 21 secs later)
00000800:00000200:0.1:1788572761.494450:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572761.494451:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 21 secs later)
00000800:00000200:0.1:1788572761.494714:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572761.494716:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 21 secs later)
00000100:00000040:1.0:1788572762.453517:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304366009, at 4304366016
00000100:00000010:1.0:1788572762.453536:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7cc0.
00000020:00000001:1.0:1788572762.901622:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572762.901626:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572762.901629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572763.029574:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572763.029581:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572763.029584:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572763.029589:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572763.029592:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572763.029595:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572763.029596:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572763.029599:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572763.029602:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572763.925581:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572763.925585:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572763.925588:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572764.053466:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572764.053473:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572764.053475:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572764.053480:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572764.053483:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572764.053486:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572764.053488:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572764.053490:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572764.053492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572765.077499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572765.077506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572765.077507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572765.077511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572765.077514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572765.077517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572765.077519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572765.077522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572765.077524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572766.101489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572766.101496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572766.101497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572766.101502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572766.101505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572766.101508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572766.101510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572766.101512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572766.101514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572766.614194:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572766.614200:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 16 secs later)
00000800:00000200:0.1:1788572766.614773:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572766.614777:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 16 secs later)
00000800:00000200:0.1:1788572766.615104:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572766.615107:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 16 secs later)
00000020:00000001:1.0:1788572766.997622:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572766.997625:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572766.997628:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572767.125550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572767.125575:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572767.125576:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572767.125583:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572767.125586:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572767.125591:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572767.125594:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572767.125596:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572767.125599:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572767.573546:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304371129, at 4304371136
00000100:00000010:1.0:1788572767.573580:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7de0.
00000020:00000001:1.0:1788572768.021611:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572768.021617:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572768.021621:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572768.149537:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572768.149545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572768.149547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572768.149553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572768.149572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572768.149576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572768.149579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572768.149581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572768.149584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572769.173521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572769.173531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572769.173534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572769.173542:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572769.173547:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572769.173553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572769.173556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572769.173576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572769.173580:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572770.197514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572770.197522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572770.197525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572770.197533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572770.197538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572770.197543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572770.197547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572770.197551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572770.197555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572771.093651:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572771.093655:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572771.093659:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572771.221593:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572771.221601:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572771.221603:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572771.221609:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572771.221613:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572771.221617:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572771.221619:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572771.221622:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572771.221624:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572771.734123:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572771.734126:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1788572771.734550:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572771.734552:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 10 secs later)
00000800:00000200:0.1:1788572771.734785:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572771.734787:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 10 secs later)
00000020:00000001:1.0:1788572772.117622:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572772.117628:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572772.117632:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572772.245518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572772.245529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572772.245532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572772.245539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572772.245543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572772.245549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572772.245552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572772.245554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572772.245557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572772.693507:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304376249, at 4304376256
00000100:00000010:1.0:1788572772.693528:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d76c0.
00000100:00000001:1.0:1788572773.269524:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572773.269532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572773.269534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572773.269539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572773.269543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572773.269547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572773.269549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572773.269552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572773.269555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572774.293611:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572774.293622:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572774.293625:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572774.293632:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572774.293637:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572774.293642:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572774.293646:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572774.293650:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572774.293653:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572775.189556:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572775.189560:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572775.189563:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572775.317468:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572775.317475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572775.317477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572775.317482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572775.317485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572775.317487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572775.317489:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572775.317491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572775.317493:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572776.213530:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572776.213536:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572776.213540:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572776.341504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572776.341510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572776.341512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572776.341516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572776.341519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572776.341522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572776.341524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572776.341526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572776.341528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572776.854223:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572776.854236:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 5 secs later)
00000800:00000200:0.1:1788572776.854636:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572776.854638:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 5 secs later)
00000800:00000200:0.1:1788572776.854869:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572776.854871:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 5 secs later)
00000100:00000001:1.0:1788572777.365463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572777.365472:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572777.365475:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572777.365481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572777.365485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572777.365490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572777.365494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572777.365498:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572777.365501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572777.813482:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304381369, at 4304381376
00000100:00000010:1.0:1788572777.813496:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ba0.
00000100:00000001:1.0:1788572778.389507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572778.389514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572778.389516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572778.389522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572778.389526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572778.389530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572778.389533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572778.389536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572778.389539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572779.285627:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572779.285632:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572779.285636:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572779.413470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572779.413477:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572779.413479:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572779.413484:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572779.413487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572779.413490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572779.413492:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572779.413495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572779.413497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572780.309542:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572780.309546:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572780.309550:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572780.437502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572780.437508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572780.437510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572780.437514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572780.437517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572780.437520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572780.437522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572780.437524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572780.437526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572781.461496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572781.461503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572781.461505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572781.461511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572781.461515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572781.461518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572781.461521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572781.461524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572781.461527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572781.973555:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572781.973567:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788572781.973570:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572781.973585:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572781.973590:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572781.973598:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 9713
00000400:00000200:1.0:1788572781.973608:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572781.973614:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572781.973617:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572781.973621:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973624:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572781.973626:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973628:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572781.973632:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572781.973633:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788572781.973636:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572781.973639:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788572781.973640:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788572781.973642:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788572781.973644:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973645:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572781.973647:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973648:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572781.973651:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788572781.973654:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973656:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572781.973658:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572781.973661:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572781.973664:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572781.973666:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788572781.973668:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572781.973680:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788572781.973685:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788572781.973689:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572781.973691:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572781.973695:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572781.973699:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572781.973705:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572781.973715:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788572781.973724:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572781.973734:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393009920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572781.973744:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572781.973748:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572781.973752:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393009920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572781.973771:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572781.973774:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572781.973776:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788572781.973778:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572781.973779:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572781.973781:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572781.973784:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9713
00000400:00000200:1.0:1788572781.973788:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572781.973792:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572781.973793:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572781.973796:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973798:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572781.973800:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973802:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572781.973804:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572781.973806:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788572781.973808:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572781.973813:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788572781.973817:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788572781.973821:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788572781.973829:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973837:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572781.973852:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973864:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572781.973876:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788572781.973889:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572781.973901:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572781.973908:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572781.973920:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572781.973927:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572781.973939:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572781.973948:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788572781.973958:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572781.973964:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572781.973973:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788572781.973982:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788572781.973989:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572781.973993:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572781.974008:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572781.974016:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572781.974024:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572781.974038:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788572781.974049:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572781.974064:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393010048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572781.974122:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572781.974131:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572781.974151:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393010048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1788572781.974176:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788572781.974191:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572781.974197:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572781.974201:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788572781.974203:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572781.974207:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572781.974215:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572781.974225:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9713
00000400:00000200:1.0:1788572781.974234:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572781.974247:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572781.974256:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572781.974268:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.974283:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572781.974292:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.974303:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572781.974312:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572781.974315:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788572781.974317:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572781.974319:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788572781.974322:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788572781.974329:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788572781.974340:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.974345:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572781.974351:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572781.974359:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572781.974367:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788572781.974377:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572781.974385:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572781.974388:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572781.974428:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572781.974432:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572781.974436:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572781.974440:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788572781.974446:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572781.974452:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572781.974470:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788572781.974479:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788572781.974491:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572781.974497:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572781.974500:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572781.974503:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572781.974506:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572781.974520:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788572781.974530:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572781.974539:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393010176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572781.974560:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572781.974566:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572781.974590:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393010176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572781.974622:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572781.974628:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788572781.974640:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788572781.974651:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (9723)
00000800:00000200:0.1:1788572781.974715:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572781.974894:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788572782.485490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572782.485496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572782.485498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572782.485502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572782.485505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572782.485508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572782.485510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572782.485512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572782.485515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572782.933514:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304386489, at 4304386496
00000100:00000010:1.0:1788572782.933537:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7b40.
00000020:00000001:1.0:1788572783.381525:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572783.381528:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572783.381530:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572783.509484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572783.509490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572783.509492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572783.509496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572783.509499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572783.509502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572783.509504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572783.509506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572783.509508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572784.405672:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572784.405675:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572784.405678:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572787.094372:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572787.094381:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788572787.095128:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572787.095131:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788572787.095502:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572787.095505:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788572787.477579:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572787.477607:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572787.477611:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572788.053513:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304391610, at 4304391616
00000100:00000010:1.0:1788572788.053531:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7180.
00000020:00000001:1.0:1788572788.501705:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572788.501712:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572788.501717:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572791.573539:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572791.573543:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572791.573545:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572792.213949:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572792.213953:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788572792.214431:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572792.214433:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788572792.214691:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572792.214692:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:1.0:1788572792.597607:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572792.597612:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572792.597616:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572793.173468:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304396729, at 4304396736
00000100:00000010:1.0:1788572793.173488:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d20.
00000800:00000200:1.0:1788572793.237736:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788572795.669539:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572795.669544:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572795.669549:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572796.693584:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572796.693588:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572796.693592:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572797.334167:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572797.334171:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788572797.334680:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572797.334683:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788572797.335002:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572797.335005:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:1.0:1788572798.293535:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304401849, at 4304401856
00000100:00000010:1.0:1788572798.293561:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7240.
00000020:00000001:1.0:1788572799.765592:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572799.765596:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572799.765610:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572800.789557:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572800.789560:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572800.789562:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1788572802.454018:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572802.454023:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788572802.454442:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572802.454445:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788572802.454673:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572802.454675:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:1.0:1788572803.413493:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304406969, at 4304406976
00000100:00000010:1.0:1788572803.413506:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000020:00000001:1.0:1788572803.861583:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572803.861586:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572803.861589:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572804.885597:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572804.885602:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572804.885622:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1788572807.574136:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572807.574141:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788572807.574477:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572807.574479:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788572807.574735:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572807.574737:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:1.0:1788572807.957607:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572807.957644:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572807.957647:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572808.533494:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304412089, at 4304412096
00000100:00000010:1.0:1788572808.533511:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000020:00000001:1.0:1788572808.981529:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572808.981533:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572808.981537:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572812.053608:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572812.053612:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572812.053629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572812.694211:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572812.694216:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788572812.694617:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572812.694619:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788572812.694854:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572812.694856:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:1.0:1788572813.077557:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572813.077562:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572813.077565:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572813.653498:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304417209, at 4304417216
00000100:00000010:1.0:1788572813.653513:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f60.
00000020:00000001:1.0:1788572816.149619:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572816.149638:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572816.149641:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572817.173567:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572817.173586:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572817.173589:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572817.814034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572817.814054:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788572817.814572:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572817.814575:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788572817.814908:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572817.814911:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:1.0:1788572818.773482:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304422329, at 4304422336
00000100:00000010:1.0:1788572818.773501:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7060.
00000020:00000001:1.0:1788572820.245529:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572820.245534:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572820.245537:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572821.269616:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572821.269620:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572821.269622:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788572822.933491:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572822.933502:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788572822.933504:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572822.933508:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572822.933512:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572822.933520:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 9754
00000400:00000200:1.0:1788572822.933529:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572822.933536:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572822.933539:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572822.933542:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933545:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572822.933547:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933548:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572822.933552:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572822.933553:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788572822.933556:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572822.933558:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788572822.933560:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788572822.933562:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788572822.933564:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933565:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572822.933567:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933568:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572822.933572:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788572822.933574:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933576:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572822.933579:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572822.933581:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572822.933583:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572822.933585:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788572822.933587:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572822.933594:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788572822.933597:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788572822.933600:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572822.933601:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572822.933603:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572822.933605:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572822.933608:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572822.933614:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788572822.933619:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572822.933626:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393012992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572822.933634:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572822.933638:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572822.933642:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393012992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572822.933658:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572822.933662:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572822.933665:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788572822.933667:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572822.933669:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572822.933672:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572822.933676:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9754
00000400:00000200:1.0:1788572822.933682:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572822.933690:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572822.933696:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572822.933704:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933710:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572822.933715:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933718:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572822.933730:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572822.933734:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788572822.933737:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572822.933740:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788572822.933744:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788572822.933749:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788572822.933753:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933755:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572822.933757:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933760:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572822.933763:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788572822.933767:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933771:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572822.933774:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572822.933786:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572822.933790:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572822.933794:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572822.933799:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788572822.933803:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572822.933807:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572822.933812:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788572822.933815:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788572822.933821:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572822.933824:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572822.933841:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572822.933843:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572822.933844:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572822.933848:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788572822.933852:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572822.933860:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393013120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572822.933873:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572822.933878:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572822.933889:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393013120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572822.933911:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572822.933916:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572822.933920:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788572822.933922:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572822.933925:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572822.933929:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572822.933935:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9754
00000400:00000200:1.0:1788572822.933941:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572822.933946:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572822.933949:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572822.933954:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.933959:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572822.933965:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:0.1:1788572822.933968:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788572822.933970:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:0.1:1788572822.933972:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788572822.933977:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572822.933980:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788572822.933986:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572822.933990:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788572822.933996:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788572822.934001:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788572822.934010:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.934014:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572822.934018:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572822.934022:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572822.934026:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788572822.934030:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572822.934033:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572822.934035:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572822.934040:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572822.934044:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572822.934048:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572822.934052:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788572822.934055:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572822.934058:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572822.934062:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788572822.934066:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788572822.934071:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572822.934074:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572822.934078:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572822.934080:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572822.934082:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572822.934088:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788572822.934093:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572822.934102:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393013248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572822.934121:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572822.934127:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572822.934137:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393013248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572822.934153:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572822.934160:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788572822.934165:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788572822.934170:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (9764)
00000800:00000200:0.1:1788572822.934417:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572822.934419:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788572822.934633:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572822.934634:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:1.0:1788572823.893505:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304427449, at 4304427456
00000100:00000010:1.0:1788572823.893522:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7540.
00000020:00000001:1.0:1788572824.341717:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572824.341722:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572824.341726:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572825.365604:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572825.365608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572825.365611:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572828.053965:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572828.053969:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788572828.054276:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572828.054278:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788572828.054513:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572828.054515:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788572828.437550:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572828.437553:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572828.437557:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572829.013500:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304432569, at 4304432576
00000100:00000010:1.0:1788572829.013515:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ea0.
00000020:00000001:1.0:1788572829.461604:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572829.461608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572829.461611:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788572831.125774:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788572832.533531:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572832.533535:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572832.533537:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788572833.173496:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572833.173508:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788572833.173511:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572833.173514:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572833.173520:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572833.173527:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 9764
00000400:00000200:1.0:1788572833.173535:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572833.173541:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572833.173544:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572833.173547:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173552:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572833.173554:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173557:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572833.173561:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572833.173563:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788572833.173566:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572833.173570:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788572833.173572:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788572833.173575:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788572833.173578:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173581:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572833.173583:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173585:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572833.173589:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788572833.173592:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173594:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572833.173597:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572833.173600:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572833.173602:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572833.173604:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788572833.173606:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572833.173613:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788572833.173616:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788572833.173620:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572833.173621:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572833.173624:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572833.173626:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572833.173629:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572833.173635:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788572833.173641:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572833.173648:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393013760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572833.173661:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572833.173665:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572833.173669:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393013760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572833.173685:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572833.173688:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572833.173690:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788572833.173692:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572833.173693:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572833.173695:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572833.173698:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9764
00000400:00000200:1.0:1788572833.173702:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572833.173706:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572833.173709:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572833.173715:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173721:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572833.173727:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173730:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572833.173736:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572833.173738:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788572833.173742:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572833.173746:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788572833.173751:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788572833.173755:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788572833.173760:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173765:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572833.173767:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173771:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572833.173776:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788572833.173782:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572833.173789:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572833.173792:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572833.173797:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572833.173802:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572833.173804:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572833.173808:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788572833.173815:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572833.173817:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572833.173822:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788572833.173826:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788572833.173837:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572833.173840:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572833.173844:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572833.173848:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572833.173854:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572833.173865:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788572833.173872:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572833.173884:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393013888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572833.173898:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572833.173903:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572833.173913:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393013888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572833.173931:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572833.173936:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572833.173942:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788572833.173961:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572833.173967:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572833.173972:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572833.173979:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9764
00000400:00000200:1.0:1788572833.173999:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572833.174011:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572833.174017:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572833.174026:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.174046:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572833.174068:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:0.1:1788572833.174068:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788572833.174072:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572833.174075:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572833.174077:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:0.1:1788572833.174077:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:1.0:1788572833.174080:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572833.174082:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788572833.174085:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788572833.174087:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788572833.174091:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.174095:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572833.174099:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572833.174101:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572833.174103:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788572833.174105:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572833.174108:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572833.174111:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572833.174114:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572833.174118:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572833.174122:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572833.174124:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788572833.174127:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572833.174132:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572833.174135:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788572833.174139:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788572833.174143:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572833.174145:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572833.174149:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572833.174152:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572833.174155:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572833.174160:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788572833.174165:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572833.174174:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393014016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572833.174185:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572833.174188:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572833.174194:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393014016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572833.174208:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572833.174211:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788572833.174215:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788572833.174219:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (9774)
00000800:00000200:0.1:1788572833.174500:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572833.174502:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788572833.174769:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572833.174772:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:1.0:1788572833.557581:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572833.557585:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572833.557588:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572834.133499:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304437689, at 4304437696
00000100:00000010:1.0:1788572834.133512:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7120.
00000100:00000001:1.0:1788572834.709495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572834.709501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572834.709503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572834.709509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572834.709514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572834.709519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572834.709522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572834.709525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572834.709528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572835.733510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572835.733525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572835.733529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572835.733552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572835.733559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572835.733565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572835.733570:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572835.733575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572835.733580:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572836.629739:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572836.629746:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572836.629752:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572836.757493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572836.757502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572836.757504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572836.757511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572836.757515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572836.757519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572836.757522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572836.757525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572836.757528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572837.653625:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572837.653628:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572837.653631:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572837.781443:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572837.781449:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572837.781450:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572837.781465:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572837.781468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572837.781471:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572837.781473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572837.781475:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572837.781477:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572838.294165:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572838.294169:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788572838.294698:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572838.294701:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788572838.294946:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572838.294947:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:1.0:1788572838.805492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572838.805498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572838.805500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572838.805504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572838.805506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572838.805509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572838.805512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572838.805514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572838.805516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572839.253457:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304442809, at 4304442816
00000100:00000010:1.0:1788572839.253471:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7120.
00000100:00000001:1.0:1788572839.829485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572839.829493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572839.829496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572839.829502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572839.829507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572839.829511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572839.829514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572839.829516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572839.829519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572840.725562:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572840.725566:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572840.725569:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572840.853490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572840.853496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572840.853498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572840.853502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572840.853504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572840.853507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572840.853510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572840.853512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572840.853514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572841.749579:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572841.749582:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572841.749586:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572841.877465:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572841.877471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572841.877473:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572841.877478:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572841.877481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572841.877483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572841.877485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572841.877487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572841.877489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572842.901503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572842.901510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572842.901512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572842.901518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572842.901522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572842.901526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572842.901528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572842.901531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572842.901534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572843.414074:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572843.414081:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788572843.414465:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572843.414467:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788572843.414687:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572843.414689:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:1.0:1788572843.925451:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572843.925457:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572843.925459:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572843.925463:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572843.925465:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572843.925469:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572843.925471:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572843.925472:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572843.925474:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572844.373502:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304447929, at 4304447936
00000100:00000010:1.0:1788572844.373518:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ea0.
00000020:00000001:1.0:1788572844.821581:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572844.821585:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572844.821588:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572844.949564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572844.949578:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572844.949580:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572844.949585:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572844.949588:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572844.949591:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572844.949592:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572844.949595:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572844.949597:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572845.845556:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572845.845560:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572845.845562:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572845.973464:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572845.973471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572845.973472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572845.973477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572845.973480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572845.973483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572845.973485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572845.973487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572845.973489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572846.997478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572846.997485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572846.997488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572846.997493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572846.997497:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572846.997501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572846.997503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572846.997506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572846.997509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572848.021487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572848.021493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572848.021494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572848.021499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572848.021502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572848.021505:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572848.021508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572848.021510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572848.021513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572848.534112:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572848.534413:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572848.534538:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788572848.917551:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572848.917554:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572848.917557:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572849.045463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572849.045469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572849.045471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572849.045476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572849.045479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572849.045482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572849.045485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572849.045487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572849.045489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572849.493462:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304453049, at 4304453056
00000100:00000010:1.0:1788572849.493472:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7540.
00000020:00000001:1.0:1788572849.941554:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572849.941557:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572849.941560:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572850.069503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572850.069510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572850.069518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572850.069523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572850.069525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572850.069528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572850.069530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572850.069533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572850.069535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572851.093496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572851.093505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572851.093507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572851.093513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572851.093516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572851.093521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572851.093524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572851.093527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572851.093530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572852.117525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572852.117531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572852.117533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572852.117539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572852.117543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572852.117548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572852.117550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572852.117553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572852.117556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572853.013608:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572853.013611:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572853.013613:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572853.141501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572853.141507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572853.141509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572853.141513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572853.141516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572853.141520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572853.141522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572853.141524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572853.141526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572853.654172:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572853.654179:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788572853.654815:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572853.654819:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788572853.655118:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572853.655121:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788572854.037651:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572854.037655:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572854.037657:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572854.165562:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572854.165575:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572854.165579:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572854.165587:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572854.165591:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572854.165595:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572854.165598:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572854.165601:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572854.165603:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572854.613558:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304458169, at 4304458176
00000100:00000010:1.0:1788572854.613574:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7060.
00000100:00000001:1.0:1788572855.189535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572855.189544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572855.189545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572855.189552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572855.189555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572855.189559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572855.189561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572855.189564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572855.189567:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572856.213508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572856.213514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572856.213516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572856.213520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572856.213524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572856.213527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572856.213530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572856.213532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572856.213534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572857.109651:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572857.109656:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572857.109660:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572857.237474:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572857.237481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572857.237482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572857.237488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572857.237490:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572857.237493:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572857.237495:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572857.237497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572857.237499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572858.133588:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572858.133593:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572858.133597:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572858.261517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572858.261539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572858.261542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572858.261548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572858.261552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572858.261556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572858.261559:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572858.261562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572858.261565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572858.774200:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572858.774208:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1788572858.774801:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572858.774803:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1788572858.775093:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572858.775096:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000001:1.0:1788572859.285472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572859.285479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572859.285481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572859.285492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572859.285495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572859.285497:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572859.285499:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572859.285501:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572859.285503:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572859.733474:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304463289, at 4304463296
00000100:00000010:1.0:1788572859.733490:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:1.0:1788572860.309475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572860.309481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572860.309483:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572860.309488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572860.309491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572860.309493:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572860.309496:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572860.309499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572860.309502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572861.205524:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572861.205529:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572861.205533:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572861.333564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572861.333572:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572861.333575:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572861.333581:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572861.333585:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572861.333603:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572861.333606:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572861.333608:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572861.333610:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572862.229539:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572862.229543:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572862.229546:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572862.357498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572862.357504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572862.357506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572862.357510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572862.357512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572862.357527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572862.357529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572862.357531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572862.357533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572863.381508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572863.381516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572863.381519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572863.381526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572863.381530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572863.381534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572863.381538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572863.381540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572863.381545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572863.894041:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572863.894047:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1788572863.894434:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572863.894436:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1788572863.894627:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572863.894629:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:1.0:1788572864.405502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572864.405508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572864.405510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572864.405514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572864.405516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572864.405520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572864.405522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572864.405524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572864.405526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572864.853481:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304468409, at 4304468416
00000100:00000010:1.0:1788572864.853505:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000020:00000001:1.0:1788572865.301578:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572865.301582:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572865.301586:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572865.429504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572865.429512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572865.429515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572865.429521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572865.429524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572865.429528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572865.429531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572865.429534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572865.429537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572866.325660:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572866.325665:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572866.325679:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572866.453505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572866.453510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572866.453512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572866.453516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572866.453519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572866.453522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572866.453524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572866.453526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572866.453528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572867.477512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572867.477518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572867.477520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572867.477524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572867.477527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572867.477530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572867.477532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572867.477534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572867.477536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572868.501498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572868.501506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572868.501508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572868.501513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572868.501516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572868.501518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572868.501521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572868.501522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572868.501524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572869.014052:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572869.014058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788572869.014463:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572869.014465:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788572869.014733:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572869.014735:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:1.0:1788572869.397544:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572869.397548:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572869.397550:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572869.525510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572869.525516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572869.525518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572869.525522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572869.525524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572869.525528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572869.525530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572869.525532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572869.525534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572869.973507:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304473529, at 4304473536
00000100:00000010:1.0:1788572869.973523:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000800:00000200:1.0:1788572870.037465:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788572870.037472:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788572870.037476:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788572870.037622:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788572870.037636:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788572870.037643:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788572870.037646:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788572870.421565:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572870.421570:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572870.421574:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572870.549495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572870.549503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572870.549505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572870.549511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572870.549515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572870.549518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572870.549521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572870.549524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572870.549526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572871.573506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572871.573512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572871.573515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572871.573521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572871.573525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572871.573529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572871.573531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572871.573534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572871.573537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572872.597508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572872.597516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572872.597519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572872.597525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572872.597528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572872.597531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572872.597534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572872.597535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572872.597538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572873.493594:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572873.493598:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572873.493602:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572873.621527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572873.621534:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572873.621535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572873.621540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572873.621543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572873.621547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572873.621549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572873.621551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572873.621553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572874.134025:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572874.134030:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788572874.134319:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572874.134321:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788572874.134552:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572874.134554:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:1.0:1788572874.517609:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572874.517613:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572874.517617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572874.645467:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572874.645474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572874.645476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572874.645482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572874.645485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572874.645490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572874.645493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572874.645495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572874.645498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572875.093496:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304478649, at 4304478656
00000100:00000010:1.0:1788572875.093511:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:1.0:1788572875.669542:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572875.669550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572875.669552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572875.669557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572875.669561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572875.669566:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572875.669569:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572875.669572:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572875.669576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572876.693554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572876.693565:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572876.693568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572876.693575:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572876.693580:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572876.693585:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572876.693589:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572876.693592:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572876.693597:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572877.589635:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572877.589639:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572877.589642:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572877.717540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572877.717549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572877.717552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572877.717557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572877.717561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572877.717563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572877.717566:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572877.717567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572877.717569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572878.613558:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572878.613561:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572878.613564:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572878.741459:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572878.741467:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572878.741470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572878.741475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572878.741479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572878.741482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572878.741485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572878.741488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572878.741491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572879.254166:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572879.254172:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1788572879.254671:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572879.254673:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1788572879.254994:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572879.254997:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:1.0:1788572879.765497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572879.765503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572879.765505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572879.765509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572879.765512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572879.765515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572879.765517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572879.765519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572879.765521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572880.213509:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304483769, at 4304483776
00000100:00000010:1.0:1788572880.213528:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000001:1.0:1788572880.789541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572880.789553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572880.789555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572880.789563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572880.789568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572880.789573:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572880.789576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572880.789580:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572880.789583:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572881.685576:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572881.685580:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572881.685583:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572881.813498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572881.813504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572881.813506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572881.813510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572881.813513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572881.813516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572881.813518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572881.813520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572881.813522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572882.709569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572882.709572:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572882.709574:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572882.837488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572882.837494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572882.837496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572882.837500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572882.837503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572882.837506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572882.837508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572882.837509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572882.837511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572883.861502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572883.861509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572883.861511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572883.861517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572883.861521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572883.861524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572883.861527:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572883.861530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572883.861533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572884.374092:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572884.374097:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788572884.374542:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572884.374545:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788572884.374799:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572884.374801:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:1.0:1788572885.333491:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304488889, at 4304488896
00000100:00000010:1.0:1788572885.333510:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000020:00000001:1.0:1788572885.781554:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572885.781557:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572885.781559:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572886.805584:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572886.805587:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572886.805589:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1788572889.493941:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572889.493945:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788572889.494298:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572889.494300:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788572889.494514:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572889.494516:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788572889.877614:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572889.877617:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572889.877620:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572890.453487:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304494009, at 4304494016
00000100:00000010:1.0:1788572890.453507:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000020:00000001:1.0:1788572890.901509:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572890.901511:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572890.901514:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572893.973555:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572893.973558:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572893.973561:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1788572894.613989:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572894.613993:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788572894.614449:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572894.614451:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788572894.614730:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572894.614732:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788572894.997574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572894.997578:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572894.997580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572895.125512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572895.125518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572895.125519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572895.125524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572895.125526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572895.125529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572895.125531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572895.125533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572895.125535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572895.573486:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304499129, at 4304499136
00000100:00000010:1.0:1788572895.573501:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:1.0:1788572896.149501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572896.149508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572896.149510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572896.149515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572896.149518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572896.149521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572896.149524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572896.149527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572896.149529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572897.173485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572897.173491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572897.173493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572897.173498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572897.173500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572897.173503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572897.173505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572897.173507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572897.173509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572898.069585:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572898.069588:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572898.069591:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572898.197461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572898.197468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572898.197470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572898.197474:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572898.197476:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572898.197479:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572898.197481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572898.197483:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572898.197485:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572899.093560:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572899.093564:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572899.093568:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572899.221510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572899.221518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572899.221520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572899.221527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572899.221531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572899.221536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572899.221540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572899.221543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572899.221546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572899.734081:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572899.734084:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788572899.734605:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572899.734608:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1788572899.734929:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572899.734932:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000100:00000001:1.0:1788572900.245518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572900.245525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572900.245526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572900.245530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572900.245533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572900.245536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572900.245538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572900.245541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572900.245543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572900.693447:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304504249, at 4304504256
00000100:00000010:1.0:1788572900.693462:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:1.0:1788572901.269439:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572901.269446:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572901.269448:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572901.269452:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572901.269455:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572901.269459:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572901.269461:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572901.269464:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572901.269467:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572902.165567:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572902.165571:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572902.165574:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572902.293448:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572902.293455:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572902.293456:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572902.293460:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572902.293463:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572902.293466:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572902.293468:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572902.293470:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572902.293472:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572903.189576:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572903.189579:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572903.189583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572903.317509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572903.317516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572903.317519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572903.317525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572903.317529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572903.317533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572903.317535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572903.317539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572903.317542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572904.341479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572904.341488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572904.341490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572904.341496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572904.341508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572904.341513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572904.341516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572904.341518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572904.341521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572904.854093:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572904.854097:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1788572904.854582:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572904.854584:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1788572904.854811:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572904.854813:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00000001:1.0:1788572905.365571:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572905.365578:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572905.365580:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572905.365584:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572905.365587:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572905.365590:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572905.365592:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572905.365594:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572905.365596:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572905.813507:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304509369, at 4304509376
00000100:00000010:1.0:1788572905.813526:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000020:00000001:1.0:1788572906.261551:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572906.261560:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572906.261563:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572906.389493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572906.389498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572906.389500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572906.389506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572906.389509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572906.389512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572906.389514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572906.389516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572906.389518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572907.285642:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572907.285646:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572907.285649:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788572907.925560:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788572907.925571:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788572907.925577:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788572907.925781:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788572907.925797:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788572907.925802:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788572907.925808:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:3.1:1788572909.974151:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572909.974155:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1788572909.974641:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572909.974643:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1788572909.974891:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572909.974893:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:1.0:1788572910.357569:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572910.357574:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572910.357579:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572910.933508:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304514489, at 4304514496
00000100:00000010:1.0:1788572910.933522:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000020:00000001:1.0:1788572911.381548:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572911.381551:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572911.381553:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572913.557515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572913.557522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572913.557524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572913.557529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572913.557533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572913.557536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572913.557539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572913.557542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572913.557545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572914.453595:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572914.453599:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572914.453602:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572914.581484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572914.581490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572914.581491:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572914.581496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572914.581498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572914.581502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572914.581504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572914.581505:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572914.581507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572915.093994:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572915.094201:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572915.094414:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788572915.477599:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572915.477602:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572915.477606:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572915.605460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572915.605466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572915.605468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572915.605472:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572915.605475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572915.605478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572915.605480:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572915.605482:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572915.605484:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572916.053479:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304519609, at 4304519616
00000100:00000010:1.0:1788572916.053497:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000001:1.0:1788572916.629499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572916.629505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572916.629507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572916.629511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572916.629514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572916.629517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572916.629519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572916.629522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572916.629524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572917.653482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572917.653487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572917.653489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572917.653494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572917.653498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572917.653503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572917.653506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572917.653508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572917.653511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572918.549559:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572918.549562:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572918.549565:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572918.677500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572918.677515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572918.677517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572918.677521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572918.677525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572918.677529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572918.677532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572918.677534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572918.677536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572919.573630:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572919.573635:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572919.573638:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572919.701463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572919.701469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572919.701470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572919.701475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572919.701478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572919.701481:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572919.701484:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572919.701485:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572919.701488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572920.214168:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572920.214172:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788572920.215623:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572920.215625:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788572920.215864:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572920.215867:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:1.0:1788572920.725489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572920.725496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572920.725497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572920.725502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572920.725506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572920.725508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572920.725511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572920.725513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572920.725516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572921.173487:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304524730, at 4304524736
00000100:00000010:1.0:1788572921.173503:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:1.0:1788572921.749515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572921.749525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572921.749528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572921.749535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572921.749540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572921.749546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572921.749549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572921.749553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572921.749557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572922.645644:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572922.645647:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572922.645651:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572922.773496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572922.773508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572922.773511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572922.773519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572922.773524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572922.773529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572922.773533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572922.773536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572922.773540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572923.669607:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572923.669613:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572923.669618:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572923.797470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572923.797476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572923.797478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572923.797482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572923.797485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572923.797488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572923.797490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572923.797492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572923.797494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572924.821532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572924.821540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572924.821542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572924.821549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572924.821559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572924.821565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572924.821569:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572924.821573:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572924.821576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572925.333581:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572925.333595:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788572925.333599:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572925.333606:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572925.333612:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572925.333623:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 9856
00000400:00000200:1.0:1788572925.333635:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572925.333645:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572925.333648:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572925.333652:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333655:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572925.333658:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333661:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572925.333666:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572925.333669:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788572925.333671:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572925.333675:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788572925.333679:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788572925.333681:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788572925.333685:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333687:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572925.333689:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333691:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572925.333694:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788572925.333698:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333702:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572925.333705:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572925.333709:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572925.333712:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572925.333715:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788572925.333718:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572925.333731:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788572925.333735:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788572925.333739:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572925.333741:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572925.333745:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572925.333748:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572925.333755:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572925.333765:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788572925.333773:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572925.333795:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393020672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572925.333807:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572925.333813:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572925.333818:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393020672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572925.333841:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572925.333845:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572925.333849:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788572925.333851:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572925.333853:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572925.333856:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572925.333859:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9856
00000400:00000200:1.0:1788572925.333863:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572925.333868:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572925.333870:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572925.333872:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333875:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572925.333878:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333880:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572925.333888:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572925.333890:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788572925.333892:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572925.333896:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788572925.333902:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788572925.333905:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788572925.333911:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333913:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572925.333916:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333920:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572925.333923:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788572925.333928:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572925.333931:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572925.333935:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572925.333943:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572925.333955:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572925.333958:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572925.333966:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788572925.333980:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572925.333997:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572925.334007:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788572925.334017:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788572925.334027:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572925.334029:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572925.334031:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572925.334032:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572925.334034:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572925.334038:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788572925.334043:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572925.334053:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393020800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572925.334074:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572925.334076:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572925.334091:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393020800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572925.334108:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572925.334114:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572925.334124:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788572925.334131:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572925.334135:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572925.334147:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572925.334151:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9856
00000400:00000200:1.0:1788572925.334163:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572925.334170:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572925.334175:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572925.334183:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.334192:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572925.334204:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.334210:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572925.334218:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572925.334221:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788572925.334229:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572925.334234:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788572925.334244:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788572925.334253:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788572925.334267:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.334277:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572925.334290:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572925.334298:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572925.334310:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788572925.334326:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572925.334336:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572925.334348:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572925.334356:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.1:1788572925.334357:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572925.334364:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
02000000:00000001:1.0:1788572925.334366:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572925.334375:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572925.334382:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788572925.334445:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572925.334453:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572925.334467:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788572925.334473:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788572925.334482:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572925.334485:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572925.334491:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572925.334496:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572925.334500:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572925.334509:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788572925.334520:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572925.334531:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393020928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572925.334557:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572925.334562:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572925.334570:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393020928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572925.334588:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572925.334591:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788572925.334597:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788572925.334608:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (9866)
00000800:00000200:3.1:1788572925.334859:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572925.334860:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788572925.335087:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572925.335089:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788572925.845587:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572925.845598:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572925.845601:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572925.845609:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572925.845614:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572925.845620:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572925.845623:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572925.845627:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572925.845630:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572926.293576:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304529850, at 4304529856
00000100:00000010:1.0:1788572926.293594:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000020:00000001:1.0:1788572926.741588:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572926.741591:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572926.741594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572926.869503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572926.869509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572926.869511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572926.869518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572926.869522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572926.869526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572926.869528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572926.869531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572926.869534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572927.765534:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572927.765539:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572927.765543:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788572930.453554:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572930.453566:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788572930.453569:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572930.453574:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572930.453579:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572930.453587:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 9861
00000400:00000200:1.0:1788572930.453597:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572930.453603:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572930.453606:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572930.453609:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453612:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572930.453614:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453616:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572930.453620:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572930.453622:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788572930.453625:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572930.453628:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788572930.453630:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788572930.453632:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788572930.453633:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453635:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572930.453636:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453638:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572930.453641:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788572930.453644:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453647:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572930.453649:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572930.453652:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572930.453655:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572930.453656:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788572930.453659:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572930.453677:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788572930.453682:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788572930.453686:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572930.453688:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572930.453692:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572930.453695:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572930.453701:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572930.453713:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788572930.453723:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572930.453735:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393021056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572930.453745:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572930.453749:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572930.453752:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393021056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572930.453774:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572930.453777:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572930.453780:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788572930.453781:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572930.453783:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572930.453786:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572930.453788:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9861
00000400:00000200:1.0:1788572930.453793:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572930.453796:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572930.453798:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572930.453800:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453802:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572930.453805:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453807:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572930.453815:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572930.453817:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788572930.453826:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572930.453840:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788572930.453846:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788572930.453858:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788572930.453878:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453886:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572930.453891:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453898:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572930.453913:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788572930.453922:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572930.453928:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788572930.453937:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572930.453945:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572930.453951:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572930.453953:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572930.453959:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788572930.453969:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572930.453972:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572930.453982:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788572930.453986:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788572930.453992:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572930.453993:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572930.453996:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572930.454003:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572930.454007:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572930.454021:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788572930.454026:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572930.454040:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393021184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572930.454061:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572930.454066:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572930.454075:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393021184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572930.454097:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788572930.454112:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788572930.454116:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788572930.454128:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788572930.454133:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788572930.454143:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788572930.454157:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9861
00000400:00000200:1.0:1788572930.454172:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788572930.454188:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788572930.454198:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788572930.454202:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.454204:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572930.454206:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.454207:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788572930.454209:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788572930.454214:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788572930.454220:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788572930.454225:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788572930.454231:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788572930.454240:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788572930.454253:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.454263:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788572930.454266:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788572930.454277:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788572930.454285:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:3.1:1788572930.454293:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:1.0:1788572930.454294:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788572930.454303:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788572930.454304:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
02000000:00000001:1.0:1788572930.454309:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788572930.454313:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788572930.454326:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572930.454337:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788572930.454347:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788572930.454358:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572930.454367:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788572930.454376:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788572930.454386:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788572930.454422:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788572930.454427:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572930.454430:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788572930.454433:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788572930.454438:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788572930.454446:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788572930.454451:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572930.454466:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393021312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788572930.454478:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788572930.454494:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572930.454500:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393021312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788572930.454521:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572930.454534:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788572930.454544:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788572930.454552:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (9871)
00000800:00000200:3.1:1788572930.454796:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572930.454798:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788572930.454988:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788572930.454990:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:1.0:1788572930.837564:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572930.837568:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572930.837572:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572931.413472:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304534970, at 4304534976
00000100:00000010:1.0:1788572931.413485:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000020:00000001:1.0:1788572931.861595:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572931.861599:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572931.861603:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572934.933624:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572934.933628:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572934.933631:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572935.574069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572935.574075:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788572935.574384:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572935.574386:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788572935.574614:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572935.574615:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:1.0:1788572935.957520:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572935.957524:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572935.957526:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572936.533507:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304540089, at 4304540096
00000100:00000010:1.0:1788572936.533527:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f60.
00000020:00000001:1.0:1788572939.029564:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572939.029569:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572939.029572:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572940.053522:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572940.053525:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572940.053528:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572940.694180:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572940.694185:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788572940.694694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572940.694697:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788572940.695013:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572940.695015:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:1.0:1788572941.653498:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304545209, at 4304545216
00000100:00000010:1.0:1788572941.653518:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7240.
00000020:00000001:1.0:1788572943.125550:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572943.125555:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572943.125559:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572944.149646:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572944.149651:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572944.149655:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788572945.813509:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788572945.813523:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788572945.813529:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788572945.813726:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788572945.813747:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788572945.813757:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788572945.813764:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788572945.813957:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788572945.813961:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788572945.814302:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788572945.814312:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788572945.814555:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788572945.814557:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000040:1.0:1788572946.773475:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304550329, at 4304550336
00000100:00000010:1.0:1788572946.773489:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000020:00000001:1.0:1788572947.221560:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572947.221563:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572947.221566:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572948.245563:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572948.245566:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572948.245569:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788572950.934054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788572950.934061:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788572950.934468:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788572950.934470:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788572950.934708:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788572950.934710:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:1.0:1788572951.317525:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572951.317528:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572951.317530:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788572951.893498:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304555449, at 4304555456
00000100:00000010:1.0:1788572951.893515:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000020:00000001:1.0:1788572952.341572:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572952.341576:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572952.341579:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788572955.413590:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572955.413595:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572955.413599:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788572956.053961:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572956.053965:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788572956.054283:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572956.054285:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788572956.054509:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572956.054511:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788572956.437624:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572956.437629:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572956.437633:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572956.565510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572956.565517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572956.565519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572956.565524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572956.565527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572956.565530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572956.565532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572956.565534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572956.565536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572957.013512:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304560569, at 4304560576
00000100:00000010:1.0:1788572957.013534:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d20.
00000100:00000001:1.0:1788572957.589526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572957.589532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572957.589534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572957.589538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572957.589541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572957.589569:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572957.589571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572957.589573:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572957.589576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572958.613457:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572958.613464:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572958.613465:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572958.613469:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572958.613472:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572958.613482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572958.613484:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572958.613485:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572958.613488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572959.509581:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572959.509585:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572959.509587:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572959.637531:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572959.637565:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572959.637567:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572959.637571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572959.637574:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572959.637577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572959.637579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572959.637581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572959.637583:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572960.533572:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572960.533576:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572960.533578:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572960.661473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572960.661479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572960.661481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572960.661485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572960.661488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572960.661491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572960.661493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572960.661495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572960.661497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572961.174079:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572961.174087:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788572961.174561:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572961.174563:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788572961.174787:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572961.174789:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:1.0:1788572961.685482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572961.685489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572961.685491:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572961.685496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572961.685499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572961.685503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572961.685505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572961.685507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572961.685509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572962.133491:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304565689, at 4304565696
00000100:00000010:1.0:1788572962.133507:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7b40.
00000100:00000001:1.0:1788572962.709505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572962.709558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572962.709560:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572962.709566:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572962.709571:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572962.709575:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572962.709578:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572962.709581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572962.709584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572963.605601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572963.605604:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572963.605607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572963.733503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572963.733511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572963.733514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572963.733548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572963.733554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572963.733558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572963.733561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572963.733564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572963.733568:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572964.629591:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572964.629594:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572964.629598:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572964.757489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572964.757498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572964.757500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572964.757504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572964.757507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572964.757510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572964.757513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572964.757515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572964.757537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572965.781583:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572965.781589:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572965.781590:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572965.781594:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572965.781597:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572965.781600:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572965.781603:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572965.781605:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572965.781607:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572966.294051:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572966.294055:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788572966.294497:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572966.294499:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788572966.294728:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572966.294730:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788572966.805490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572966.805497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572966.805499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572966.805503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572966.805505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572966.805534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572966.805537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572966.805539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572966.805541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572967.253489:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304570809, at 4304570816
00000100:00000010:1.0:1788572967.253499:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ba0.
00000020:00000001:1.0:1788572967.701623:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572967.701628:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572967.701631:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572967.829473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572967.829481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572967.829484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572967.829489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572967.829493:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572967.829497:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572967.829500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572967.829503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572967.829519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572968.725557:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572968.725560:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572968.725563:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572968.853486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572968.853492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572968.853494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572968.853499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572968.853524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572968.853527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572968.853529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572968.853531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572968.853534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572969.877470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572969.877476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572969.877478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572969.877482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572969.877485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572969.877488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572969.877490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572969.877492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572969.877494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572970.901524:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572970.901532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572970.901535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572970.901541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572970.901545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572970.901549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572970.901553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572970.901556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572970.901559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572971.414027:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572971.414031:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788572971.414355:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572971.414357:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788572971.414668:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572971.414670:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:1.0:1788572971.797549:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572971.797553:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572971.797556:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572971.925455:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572971.925461:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572971.925462:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572971.925467:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572971.925470:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572971.925473:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572971.925475:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572971.925477:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572971.925479:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572972.373485:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304575929, at 4304575936
00000100:00000010:1.0:1788572972.373519:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d76c0.
00000020:00000001:1.0:1788572972.821621:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572972.821627:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572972.821632:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572972.949557:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572972.949567:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572972.949570:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572972.949577:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572972.949582:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572972.949587:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572972.949590:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572972.949594:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572972.949597:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572973.973558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572973.973567:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572973.973569:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572973.973575:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572973.973578:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572973.973582:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572973.973585:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572973.973587:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572973.973590:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572974.997509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572974.997538:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572974.997541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572974.997546:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572974.997548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572974.997551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572974.997554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572974.997556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572974.997558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572975.893678:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572975.893682:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572975.893686:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572976.021601:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572976.021611:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572976.021613:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572976.021618:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572976.021621:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572976.021625:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572976.021627:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572976.021629:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572976.021632:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572976.534132:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572976.534136:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788572976.534694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572976.534696:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788572976.534932:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572976.534934:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:1.0:1788572976.917605:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572976.917610:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572976.917614:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572977.045512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572977.045520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572977.045523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572977.045528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572977.045532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572977.045535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572977.045538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572977.045540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572977.045543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572977.493522:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304581049, at 4304581056
00000100:00000010:1.0:1788572977.493539:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7de0.
00000100:00000001:1.0:1788572978.069491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572978.069499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572978.069501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572978.069505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572978.069509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572978.069512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572978.069515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572978.069519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572978.069521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572979.093502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572979.093508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572979.093510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572979.093513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572979.093516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572979.093520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572979.093522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572979.093524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572979.093526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572979.989572:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572979.989576:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572979.989579:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572980.117485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572980.117491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572980.117493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572980.117497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572980.117500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572980.117503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572980.117505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572980.117508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572980.117510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572981.013585:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572981.013589:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572981.013592:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572981.141527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572981.141533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572981.141535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572981.141539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572981.141542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572981.141545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572981.141553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572981.141555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572981.141557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572981.654123:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572981.654369:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572981.654558:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788572982.165467:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572982.165474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572982.165476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572982.165481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572982.165484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572982.165487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572982.165489:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572982.165491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572982.165493:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572982.613571:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304586169, at 4304586176
00000100:00000010:1.0:1788572982.613591:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7cc0.
00000100:00000001:1.0:1788572983.189509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572983.189516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572983.189517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572983.189522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572983.189525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572983.189527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572983.189529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572983.189532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572983.189534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572984.085592:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572984.085597:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572984.085609:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572984.213510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572984.213516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572984.213518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572984.213522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572984.213525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572984.213528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572984.213530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572984.213532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572984.213534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572985.109606:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572985.109610:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572985.109614:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572985.237496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572985.237503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572985.237505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572985.237511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572985.237519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572985.237522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572985.237525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572985.237527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572985.237529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788572985.749523:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788572985.749529:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788572985.749534:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:1.0:1788572986.261533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572986.261540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572986.261542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572986.261547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572986.261550:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572986.261553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572986.261555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572986.261557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572986.261559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572986.774149:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572986.774154:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:0.1:1788572986.774688:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572986.774691:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:0.1:1788572986.775008:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572986.775011:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000100:00000001:1.0:1788572987.285519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572987.285528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572987.285532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572987.285539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572987.285543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572987.285547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572987.285550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572987.285552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572987.285555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572987.733509:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304591289, at 4304591296
00000100:00000010:1.0:1788572987.733528:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7c00.
00000020:00000001:1.0:1788572988.181583:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572988.181587:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572988.181590:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572988.309509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572988.309516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572988.309519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572988.309525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572988.309529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572988.309533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572988.309536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572988.309539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572988.309542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572989.205633:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572989.205637:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572989.205641:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572989.333472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572989.333480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572989.333483:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572989.333489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572989.333492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572989.333496:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572989.333499:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572989.333503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572989.333506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572990.357507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572990.357516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572990.357518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572990.357524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572990.357528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572990.357532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572990.357534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572990.357538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572990.357541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572991.381498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572991.381507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572991.381510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572991.381516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572991.381520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572991.381524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572991.381526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572991.381528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572991.381530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788572991.894358:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572991.894360:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1788572991.894636:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572991.894637:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1788572991.894880:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788572991.894881:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000020:00000001:1.0:1788572992.277561:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572992.277565:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572992.277568:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572992.405496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572992.405502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572992.405504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572992.405509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572992.405512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572992.405515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572992.405517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572992.405519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572992.405521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572992.853513:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304596410, at 4304596416
00000100:00000010:1.0:1788572992.853530:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000020:00000001:1.0:1788572993.301530:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572993.301534:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572993.301536:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572993.429525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572993.429533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572993.429536:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572993.429543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572993.429547:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572993.429551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572993.429553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572993.429555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572993.429557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572994.453529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572994.453535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572994.453537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572994.453541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572994.453544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572994.453546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572994.453549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572994.453551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572994.453555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572995.477540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572995.477549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572995.477551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572995.477557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572995.477561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572995.477565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572995.477568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572995.477570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572995.477573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788572996.373538:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572996.373542:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572996.373546:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572996.501506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572996.501514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572996.501517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572996.501523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572996.501526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572996.501530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572996.501532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572996.501534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572996.501536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788572997.014115:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788572997.014119:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788572997.014674:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788572997.014676:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788572997.014879:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788572997.014881:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:1.0:1788572997.397598:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788572997.397601:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788572997.397604:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788572997.525505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572997.525514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572997.525516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572997.525522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572997.525526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572997.525530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572997.525534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572997.525537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572997.525540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788572997.973495:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304601529, at 4304601536
00000100:00000010:1.0:1788572997.973514:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000100:00000001:1.0:1788572998.549525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572998.549531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572998.549533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572998.549538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572998.549541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572998.549544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572998.549546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572998.549548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572998.549551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788572999.573499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788572999.573508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788572999.573511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788572999.573517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788572999.573522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788572999.573526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572999.573529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788572999.573533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788572999.573538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573000.469658:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573000.469662:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573000.469666:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573000.597511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573000.597519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573000.597522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573000.597528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573000.597532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573000.597536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573000.597538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573000.597541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573000.597544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573001.493581:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573001.493584:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573001.493587:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573001.621529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573001.621538:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573001.621540:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573001.621545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573001.621549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573001.621553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573001.621555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573001.621558:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573001.621561:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573002.134217:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573002.134221:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788573002.134708:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573002.134710:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788573002.134911:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573002.134913:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:1.0:1788573002.645496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573002.645502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573002.645504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573002.645508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573002.645512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573002.645515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573002.645517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573002.645519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573002.645521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573003.093497:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304606649, at 4304606656
00000100:00000010:1.0:1788573003.093510:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d78a0.
00000100:00000001:1.0:1788573003.669494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573003.669500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573003.669502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573003.669506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573003.669509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573003.669517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573003.669519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573003.669521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573003.669523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573004.565566:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573004.565569:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573004.565572:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573004.693479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573004.693485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573004.693486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573004.693491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573004.693494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573004.693497:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573004.693499:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573004.693501:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573004.693503:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573005.589626:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573005.589631:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573005.589635:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573005.717498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573005.717504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573005.717506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573005.717510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573005.717513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573005.717517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573005.717520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573005.717522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573005.717525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573006.741436:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573006.741442:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573006.741443:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573006.741447:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573006.741450:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573006.741453:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573006.741456:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573006.741458:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573006.741460:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573007.254017:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573007.254020:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788573007.254313:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573007.254316:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788573007.254613:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573007.254615:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:1.0:1788573007.765459:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573007.765465:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573007.765467:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573007.765471:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573007.765473:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573007.765477:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573007.765479:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573007.765481:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573007.765483:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573008.213489:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304611769, at 4304611776
00000100:00000010:1.0:1788573008.213504:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7a80.
00000020:00000001:1.0:1788573008.661584:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573008.661587:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573008.661590:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573008.789449:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573008.789456:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573008.789457:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573008.789462:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573008.789464:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573008.789468:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573008.789470:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573008.789472:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573008.789473:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573009.685547:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573009.685550:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573009.685553:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573009.813485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573009.813493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573009.813495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573009.813501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573009.813505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573009.813509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573009.813512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573009.813515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573009.813518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573010.837474:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573010.837484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573010.837486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573010.837494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573010.837498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573010.837502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573010.837505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573010.837509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573010.837512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573011.861490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573011.861496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573011.861498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573011.861502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573011.861505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573011.861508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573011.861510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573011.861511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573011.861513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573012.374056:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573012.374061:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573012.374489:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573012.374491:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573012.374697:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573012.374699:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:1.0:1788573012.757527:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573012.757531:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573012.757533:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573012.885500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573012.885508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573012.885519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573012.885525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573012.885529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573012.885533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573012.885536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573012.885539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573012.885542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573013.333460:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304616889, at 4304616896
00000100:00000010:1.0:1788573013.333478:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788573013.781574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573013.781577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573013.781581:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573016.853620:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573016.853626:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573016.853629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573017.494147:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573017.494168:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573017.494669:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573017.494672:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573017.494996:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573017.495000:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:1.0:1788573017.877596:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573017.877601:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573017.877605:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573018.453509:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304622009, at 4304622016
00000100:00000010:1.0:1788573018.453528:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f00.
00000020:00000001:1.0:1788573020.949556:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573020.949559:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573020.949562:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573021.973574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573021.973577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573021.973580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573022.614190:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573022.614199:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573022.614840:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573022.614846:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573022.615093:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573022.615094:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:1.0:1788573023.573543:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304627130, at 4304627136
00000100:00000010:1.0:1788573023.573565:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7600.
00000800:00000200:1.0:1788573023.637730:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788573023.637736:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573023.637748:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573023.637754:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573023.637756:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788573025.045643:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573025.045648:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573025.045652:0:179192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 8 locks of 0 total in blp.  (0 blwis in pool)
00010000:00000001:1.0:1788573025.045658:0:179192:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered
00010000:00000001:1.0:1788573025.045661:0:179192:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered
00010000:00010000:1.0:1788573025.045666:0:179192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895343a49600/0xdb00379715726274 lrc: 2/0,0 mode: CR/CR res: [0x200002343:0x233a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc27023d919f5426f expref: -99 pid: 215613 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0
00000080:00000001:1.0:1788573025.045677:0:179192:0:(namei.c:477:ll_md_blocking_ast()) Process entered
00000080:00000001:1.0:1788573025.045681:0:179192:0:(namei.c:240:ll_lock_cancel_bits()) Process entered
00000080:00000001:1.0:1788573025.045683:0:179192:0:(file.c:6096:ll_have_md_lock()) Process entered
00000080:00000040:1.0:1788573025.045685:0:179192:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002343:0x233a:0x0] mode (null)
00000002:00000001:1.0:1788573025.045690:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.045693:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.045716:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895342de0900 count: 2
00010000:00000001:1.0:1788573025.045720:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.045724:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.045727:0:179192:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.045729:0:179192:0:(file.c:6808:ll_layout_conf()) Process entered
00000020:00000001:1.0:1788573025.045732:0:179192:0:(cl_object.c:754:cl_env_obtain()) Process entered
00000020:00000001:1.0:1788573025.045736:0:179192:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446613590345126904 : -130483364424712 : ffff8953796a13f8)
00000020:00001000:1.0:1788573025.045739:0:179192:0:(cl_object.c:823:cl_env_get()) 1@ffff8953796a13f0
00000020:00000001:1.0:1788573025.045741:0:179192:0:(cl_object.c:343:cl_conf_set()) Process entered
00000080:00200000:1.0:1788573025.045745:0:179192:0:(vvp_object.c:117:vvp_conf_set()) [0x200002343:0x233a:0x0]: losing layout lock
00000080:00000001:1.0:1788573025.045748:0:179192:0:(pcc.c:2252:pcc_layout_invalidate()) Process entered
00000080:00000001:1.0:1788573025.045751:0:179192:0:(pcc.c:2264:pcc_layout_invalidate()) Process leaving
00020000:00000001:1.0:1788573025.045754:0:179192:0:(lov_object.c:1448:lov_conf_set()) Process entered
00020000:00000001:1.0:1788573025.045757:0:179192:0:(lov_object.c:1462:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0)
00020000:00000002:1.0:1788573025.045759:0:179192:0:(lov_object.c:1535:lov_conf_set()) [0x200002343:0x233a:0x0] lo_layout_invalid=1
00020000:00000001:1.0:1788573025.045762:0:179192:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573025.045764:0:179192:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00001000:1.0:1788573025.045766:0:179192:0:(cl_object.c:917:cl_env_put()) 1@ffff8953796a13f0
00000080:00000001:1.0:1788573025.045769:0:179192:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.045771:0:179192:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving
00000080:00000001:1.0:1788573025.045772:0:179192:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1788573025.045775:0:179192:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered
00010000:00000001:1.0:1788573025.045777:0:179192:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered
00010000:00000001:1.0:1788573025.045778:0:179192:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered
00010000:00000001:1.0:1788573025.045780:0:179192:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered
00010000:00000001:1.0:1788573025.045781:0:179192:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788573025.045785:0:179192:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff895343a49600 with handle 0xdb00379715726274 from hash
00010000:00000001:1.0:1788573025.045796:0:179192:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving
00010000:00000001:1.0:1788573025.045797:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.045798:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.045799:0:179192:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving
00010000:00000001:1.0:1788573025.045802:0:179192:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving
00010000:00000001:1.0:1788573025.045803:0:179192:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000)
00010000:00000001:1.0:1788573025.045805:0:179192:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered
00010000:00010000:1.0:1788573025.045808:0:179192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c00800/0xdb00379715726258 lrc: 2/0,0 mode: PR/PR res: [0x200002343:0x2339:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc27023d919f5421b expref: -99 pid: 213723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00000080:00000001:1.0:1788573025.045817:0:179192:0:(namei.c:477:ll_md_blocking_ast()) Process entered
00000080:00000001:1.0:1788573025.045819:0:179192:0:(namei.c:240:ll_lock_cancel_bits()) Process entered
00000080:00000001:1.0:1788573025.045820:0:179192:0:(file.c:6096:ll_have_md_lock()) Process entered
00000080:00000040:1.0:1788573025.045821:0:179192:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002343:0x2339:0x0] mode (null)
00000002:00000001:1.0:1788573025.045823:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.045824:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.045826:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56100 count: 3
00010000:00000001:1.0:1788573025.045828:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.045830:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.045831:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.045832:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.045833:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56100 count: 3
00010000:00000001:1.0:1788573025.045834:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.045836:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.045837:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.045837:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.045838:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56100 count: 3
00010000:00000001:1.0:1788573025.045839:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.045841:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.045841:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.045842:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.045843:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56100 count: 3
00010000:00000001:1.0:1788573025.045844:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.045845:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.045846:0:179192:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.045847:0:179192:0:(file.c:6808:ll_layout_conf()) Process entered
00000020:00000001:1.0:1788573025.045848:0:179192:0:(cl_object.c:754:cl_env_obtain()) Process entered
00000020:00000001:1.0:1788573025.045849:0:179192:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446613590345126904 : -130483364424712 : ffff8953796a13f8)
00000020:00001000:1.0:1788573025.045851:0:179192:0:(cl_object.c:823:cl_env_get()) 1@ffff8953796a13f0
00000020:00000001:1.0:1788573025.045852:0:179192:0:(cl_object.c:343:cl_conf_set()) Process entered
00000080:00200000:1.0:1788573025.045853:0:179192:0:(vvp_object.c:117:vvp_conf_set()) [0x200002343:0x2339:0x0]: losing layout lock
00000080:00000001:1.0:1788573025.045855:0:179192:0:(pcc.c:2252:pcc_layout_invalidate()) Process entered
00000080:00000001:1.0:1788573025.045856:0:179192:0:(pcc.c:2264:pcc_layout_invalidate()) Process leaving
00020000:00000001:1.0:1788573025.045857:0:179192:0:(lov_object.c:1448:lov_conf_set()) Process entered
00020000:00000001:1.0:1788573025.045858:0:179192:0:(lov_object.c:1462:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0)
00020000:00000002:1.0:1788573025.045860:0:179192:0:(lov_object.c:1535:lov_conf_set()) [0x200002343:0x2339:0x0] lo_layout_invalid=1
00020000:00000001:1.0:1788573025.045862:0:179192:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573025.045863:0:179192:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00001000:1.0:1788573025.045865:0:179192:0:(cl_object.c:917:cl_env_put()) 1@ffff8953796a13f0
00000080:00000001:1.0:1788573025.045866:0:179192:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.045883:0:179192:0:(dcache.c:195:ll_prune_aliases()) Process entered
00000080:00000002:1.0:1788573025.045886:0:179192:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002343:0x2339:0x0](ffff89534606b248) invalid
00000020:00000001:1.0:1788573025.045890:0:179192:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff987001c47ca0
00000080:00002000:1.0:1788573025.045893:0:179192:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry f812b.sanity (ffff8953532c0288) parent ffff89534219ca20 inode ffff89534606b248 refc 0
00000080:00000001:1.0:1788573025.045902:0:179192:0:(dcache.c:39:ll_release()) Process entered
00000080:00000001:1.0:1788573025.045906:0:179192:0:(dcache.c:48:ll_release()) Process leaving
00000080:00000001:1.0:1788573025.045910:0:179192:0:(dcache.c:209:ll_prune_aliases()) Process leaving
00000080:00000001:1.0:1788573025.045913:0:179192:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving
00000080:00000001:1.0:1788573025.045914:0:179192:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1788573025.045916:0:179192:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered
00010000:00000001:1.0:1788573025.045917:0:179192:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered
00010000:00000001:1.0:1788573025.045918:0:179192:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered
00010000:00000001:1.0:1788573025.045920:0:179192:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered
00010000:00000001:1.0:1788573025.045921:0:179192:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788573025.045923:0:179192:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff895372c00800 with handle 0xdb00379715726258 from hash
00010000:00000001:1.0:1788573025.045925:0:179192:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving
00010000:00000001:1.0:1788573025.045926:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.045926:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.045927:0:179192:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving
00010000:00000001:1.0:1788573025.045928:0:179192:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving
00010000:00000001:1.0:1788573025.045929:0:179192:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000)
00010000:00000001:1.0:1788573025.045931:0:179192:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered
00010000:00010000:1.0:1788573025.045933:0:179192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c00e00/0xdb0037971572624a lrc: 2/0,0 mode: CR/CR res: [0x200002343:0x2339:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc27023d919f541f1 expref: -99 pid: 213721 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00000080:00000001:1.0:1788573025.045940:0:179192:0:(namei.c:477:ll_md_blocking_ast()) Process entered
00000080:00000001:1.0:1788573025.045942:0:179192:0:(namei.c:240:ll_lock_cancel_bits()) Process entered
00000080:00000001:1.0:1788573025.045943:0:179192:0:(file.c:6096:ll_have_md_lock()) Process entered
00000080:00000040:1.0:1788573025.045944:0:179192:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002343:0x2339:0x0] mode (null)
00000002:00000001:1.0:1788573025.045947:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.045954:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.045956:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56100 count: 3
00010000:00000001:1.0:1788573025.045957:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.045959:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.045960:0:179192:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.045962:0:179192:0:(file.c:6808:ll_layout_conf()) Process entered
00000020:00000001:1.0:1788573025.045963:0:179192:0:(cl_object.c:754:cl_env_obtain()) Process entered
00000020:00000001:1.0:1788573025.045964:0:179192:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446613590345126904 : -130483364424712 : ffff8953796a13f8)
00000020:00001000:1.0:1788573025.045965:0:179192:0:(cl_object.c:823:cl_env_get()) 1@ffff8953796a13f0
00000020:00000001:1.0:1788573025.045966:0:179192:0:(cl_object.c:343:cl_conf_set()) Process entered
00000080:00200000:1.0:1788573025.045967:0:179192:0:(vvp_object.c:117:vvp_conf_set()) [0x200002343:0x2339:0x0]: losing layout lock
00000080:00000001:1.0:1788573025.045969:0:179192:0:(pcc.c:2252:pcc_layout_invalidate()) Process entered
00000080:00000001:1.0:1788573025.045970:0:179192:0:(pcc.c:2264:pcc_layout_invalidate()) Process leaving
00020000:00000001:1.0:1788573025.045972:0:179192:0:(lov_object.c:1448:lov_conf_set()) Process entered
00020000:00000001:1.0:1788573025.045973:0:179192:0:(lov_object.c:1462:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0)
00020000:00000002:1.0:1788573025.045974:0:179192:0:(lov_object.c:1535:lov_conf_set()) [0x200002343:0x2339:0x0] lo_layout_invalid=1
00020000:00000001:1.0:1788573025.045976:0:179192:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573025.045977:0:179192:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00001000:1.0:1788573025.045978:0:179192:0:(cl_object.c:917:cl_env_put()) 1@ffff8953796a13f0
00000080:00000001:1.0:1788573025.045979:0:179192:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.045981:0:179192:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving
00000080:00000001:1.0:1788573025.045981:0:179192:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1788573025.045983:0:179192:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered
00010000:00000001:1.0:1788573025.045984:0:179192:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered
00010000:00000001:1.0:1788573025.045985:0:179192:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered
00010000:00000001:1.0:1788573025.045986:0:179192:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered
00010000:00000001:1.0:1788573025.045987:0:179192:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788573025.045989:0:179192:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff895372c00e00 with handle 0xdb0037971572624a from hash
00010000:00000001:1.0:1788573025.045990:0:179192:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving
00010000:00000001:1.0:1788573025.045991:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.045992:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.045993:0:179192:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving
00010000:00000001:1.0:1788573025.045995:0:179192:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving
00010000:00000001:1.0:1788573025.045997:0:179192:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000)
00010000:00000001:1.0:1788573025.045999:0:179192:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered
00010000:00010000:1.0:1788573025.046002:0:179192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c00c00/0xdb00379715726227 lrc: 2/0,0 mode: PR/PR res: [0x200002343:0x2338:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc27023d919f54196 expref: -99 pid: 212267 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0
00000080:00000001:1.0:1788573025.046009:0:179192:0:(namei.c:477:ll_md_blocking_ast()) Process entered
00000080:00000001:1.0:1788573025.046011:0:179192:0:(namei.c:240:ll_lock_cancel_bits()) Process entered
00000080:00000001:1.0:1788573025.046024:0:179192:0:(file.c:6096:ll_have_md_lock()) Process entered
00000080:00000040:1.0:1788573025.046026:0:179192:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002343:0x2338:0x0] mode (null)
00000002:00000001:1.0:1788573025.046028:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046030:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046032:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56800 count: 3
00010000:00000001:1.0:1788573025.046034:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046036:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.046037:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046038:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046039:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56800 count: 3
00010000:00000001:1.0:1788573025.046041:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046042:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.046043:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046044:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046045:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56800 count: 3
00010000:00000001:1.0:1788573025.046046:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046048:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.046048:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046049:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046050:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56800 count: 3
00010000:00000001:1.0:1788573025.046051:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046057:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046058:0:179192:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046059:0:179192:0:(file.c:6808:ll_layout_conf()) Process entered
00000020:00000001:1.0:1788573025.046060:0:179192:0:(cl_object.c:754:cl_env_obtain()) Process entered
00000020:00000001:1.0:1788573025.046061:0:179192:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446613590345126904 : -130483364424712 : ffff8953796a13f8)
00000020:00001000:1.0:1788573025.046062:0:179192:0:(cl_object.c:823:cl_env_get()) 1@ffff8953796a13f0
00000020:00000001:1.0:1788573025.046063:0:179192:0:(cl_object.c:343:cl_conf_set()) Process entered
00000080:00200000:1.0:1788573025.046065:0:179192:0:(vvp_object.c:117:vvp_conf_set()) [0x200002343:0x2338:0x0]: losing layout lock
00000080:00000001:1.0:1788573025.046067:0:179192:0:(pcc.c:2252:pcc_layout_invalidate()) Process entered
00000080:00000001:1.0:1788573025.046069:0:179192:0:(pcc.c:2264:pcc_layout_invalidate()) Process leaving
00020000:00000001:1.0:1788573025.046071:0:179192:0:(lov_object.c:1448:lov_conf_set()) Process entered
00020000:00000001:1.0:1788573025.046073:0:179192:0:(lov_object.c:1462:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0)
00020000:00000002:1.0:1788573025.046074:0:179192:0:(lov_object.c:1535:lov_conf_set()) [0x200002343:0x2338:0x0] lo_layout_invalid=1
00020000:00000001:1.0:1788573025.046076:0:179192:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573025.046077:0:179192:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00001000:1.0:1788573025.046078:0:179192:0:(cl_object.c:917:cl_env_put()) 1@ffff8953796a13f0
00000080:00000001:1.0:1788573025.046080:0:179192:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046081:0:179192:0:(dcache.c:195:ll_prune_aliases()) Process entered
00000080:00000002:1.0:1788573025.046082:0:179192:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002343:0x2338:0x0](ffff895351a77448) invalid
00000020:00000001:1.0:1788573025.046084:0:179192:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff987001c47ca0
00000080:00002000:1.0:1788573025.046085:0:179192:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry f812a.sanity (ffff89534962e948) parent ffff89534219ca20 inode ffff895351a77448 refc 0
00000080:00000001:1.0:1788573025.046089:0:179192:0:(dcache.c:39:ll_release()) Process entered
00000080:00000001:1.0:1788573025.046091:0:179192:0:(dcache.c:48:ll_release()) Process leaving
00000080:00000001:1.0:1788573025.046093:0:179192:0:(dcache.c:209:ll_prune_aliases()) Process leaving
00000080:00000001:1.0:1788573025.046094:0:179192:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving
00000080:00000001:1.0:1788573025.046095:0:179192:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1788573025.046097:0:179192:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered
00010000:00000001:1.0:1788573025.046098:0:179192:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered
00010000:00000001:1.0:1788573025.046099:0:179192:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered
00010000:00000001:1.0:1788573025.046101:0:179192:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered
00010000:00000001:1.0:1788573025.046103:0:179192:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788573025.046105:0:179192:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff895372c00c00 with handle 0xdb00379715726227 from hash
00010000:00000001:1.0:1788573025.046108:0:179192:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving
00010000:00000001:1.0:1788573025.046110:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.046111:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046112:0:179192:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving
00010000:00000001:1.0:1788573025.046113:0:179192:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving
00010000:00000001:1.0:1788573025.046114:0:179192:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000)
00010000:00000001:1.0:1788573025.046116:0:179192:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered
00010000:00010000:1.0:1788573025.046118:0:179192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c01400/0xdb00379715726219 lrc: 2/0,0 mode: CR/CR res: [0x200002343:0x2338:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc27023d919f5416c expref: -99 pid: 212265 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00000080:00000001:1.0:1788573025.046123:0:179192:0:(namei.c:477:ll_md_blocking_ast()) Process entered
00000080:00000001:1.0:1788573025.046124:0:179192:0:(namei.c:240:ll_lock_cancel_bits()) Process entered
00000080:00000001:1.0:1788573025.046125:0:179192:0:(file.c:6096:ll_have_md_lock()) Process entered
00000080:00000040:1.0:1788573025.046126:0:179192:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002343:0x2338:0x0] mode (null)
00000002:00000001:1.0:1788573025.046127:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046128:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046130:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56800 count: 3
00010000:00000001:1.0:1788573025.046131:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046133:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046134:0:179192:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046136:0:179192:0:(file.c:6808:ll_layout_conf()) Process entered
00000020:00000001:1.0:1788573025.046138:0:179192:0:(cl_object.c:754:cl_env_obtain()) Process entered
00000020:00000001:1.0:1788573025.046139:0:179192:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446613590345126904 : -130483364424712 : ffff8953796a13f8)
00000020:00001000:1.0:1788573025.046142:0:179192:0:(cl_object.c:823:cl_env_get()) 1@ffff8953796a13f0
00000020:00000001:1.0:1788573025.046143:0:179192:0:(cl_object.c:343:cl_conf_set()) Process entered
00000080:00200000:1.0:1788573025.046144:0:179192:0:(vvp_object.c:117:vvp_conf_set()) [0x200002343:0x2338:0x0]: losing layout lock
00000080:00000001:1.0:1788573025.046145:0:179192:0:(pcc.c:2252:pcc_layout_invalidate()) Process entered
00000080:00000001:1.0:1788573025.046146:0:179192:0:(pcc.c:2264:pcc_layout_invalidate()) Process leaving
00020000:00000001:1.0:1788573025.046147:0:179192:0:(lov_object.c:1448:lov_conf_set()) Process entered
00020000:00000001:1.0:1788573025.046148:0:179192:0:(lov_object.c:1462:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0)
00020000:00000002:1.0:1788573025.046149:0:179192:0:(lov_object.c:1535:lov_conf_set()) [0x200002343:0x2338:0x0] lo_layout_invalid=1
00020000:00000001:1.0:1788573025.046151:0:179192:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573025.046152:0:179192:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00001000:1.0:1788573025.046153:0:179192:0:(cl_object.c:917:cl_env_put()) 1@ffff8953796a13f0
00000080:00000001:1.0:1788573025.046154:0:179192:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046156:0:179192:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving
00000080:00000001:1.0:1788573025.046156:0:179192:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1788573025.046158:0:179192:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered
00010000:00000001:1.0:1788573025.046159:0:179192:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered
00010000:00000001:1.0:1788573025.046160:0:179192:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered
00010000:00000001:1.0:1788573025.046166:0:179192:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered
00010000:00000001:1.0:1788573025.046168:0:179192:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788573025.046169:0:179192:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff895372c01400 with handle 0xdb00379715726219 from hash
00010000:00000001:1.0:1788573025.046170:0:179192:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving
00010000:00000001:1.0:1788573025.046171:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.046172:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046173:0:179192:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving
00010000:00000001:1.0:1788573025.046175:0:179192:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving
00010000:00000001:1.0:1788573025.046176:0:179192:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000)
00010000:00000001:1.0:1788573025.046178:0:179192:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered
00010000:00010000:1.0:1788573025.046181:0:179192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895343a49400/0xdb003797157261f6 lrc: 2/0,0 mode: PR/PR res: [0x200002343:0x231d:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc27023d919f540fc expref: -99 pid: 211547 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0
00000080:00000001:1.0:1788573025.046188:0:179192:0:(namei.c:477:ll_md_blocking_ast()) Process entered
00000080:00000001:1.0:1788573025.046190:0:179192:0:(namei.c:240:ll_lock_cancel_bits()) Process entered
00000080:00000001:1.0:1788573025.046191:0:179192:0:(file.c:6096:ll_have_md_lock()) Process entered
00000080:00000040:1.0:1788573025.046192:0:179192:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002343:0x231d:0x0] mode (null)
00000002:00000001:1.0:1788573025.046193:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046194:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046196:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56600 count: 4
00010000:00000001:1.0:1788573025.046197:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046199:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.046201:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046202:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046204:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56600 count: 4
00010000:00000001:1.0:1788573025.046206:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00010000:00000001:1.0:1788573025.046209:0:179192:0:(ldlm_lock.c:317:ldlm_lock_touch_in_lru()) Process entered
00010000:00000001:1.0:1788573025.046212:0:179192:0:(ldlm_lock.c:330:ldlm_lock_touch_in_lru()) Process leaving
02000000:00000001:1.0:1788573025.046214:0:179192:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered
02000000:00000001:1.0:1788573025.046219:0:179192:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573025.046221:0:179192:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c01600/0xdb00379715725d8f lrc: 2/0,0 mode: CR/CR res: [0x200002343:0x231d:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xc27023d919f53269 expref: -99 pid: 211431 timeout: 0 lvb_type: 3 lru_score: 88 lru_type: 1
00010000:00000001:1.0:1788573025.046228:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.046230:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00000002:00000001:1.0:1788573025.046231:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=16 : 16 : 10)
00010000:00000001:1.0:1788573025.046233:0:179192:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered
00000020:00000001:1.0:1788573025.046235:0:179192:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573025.046237:0:179192:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff895372c01600 refcount=2
00000020:00000001:1.0:1788573025.046239:0:179192:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590233323008 : -130483476228608 : ffff895372c01600)
00010000:00000001:1.0:1788573025.046241:0:179192:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446613590233323008 : -130483476228608 : ffff895372c01600)
00010000:00000001:1.0:1788573025.046242:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.046243:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00000002:00000001:1.0:1788573025.046244:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046244:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046246:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895343c56600 count: 4
00010000:00000001:1.0:1788573025.046247:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046248:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046249:0:179192:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046251:0:179192:0:(dcache.c:195:ll_prune_aliases()) Process entered
00000080:00000002:1.0:1788573025.046252:0:179192:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002343:0x231d:0x0](ffff895351a71988) invalid
00000080:00000001:1.0:1788573025.046254:0:179192:0:(dcache.c:209:ll_prune_aliases()) Process leaving
00000080:00000001:1.0:1788573025.046256:0:179192:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving
00000080:00000001:1.0:1788573025.046257:0:179192:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1788573025.046260:0:179192:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered
00010000:00000001:1.0:1788573025.046261:0:179192:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered
00010000:00000001:1.0:1788573025.046262:0:179192:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered
00010000:00000001:1.0:1788573025.046263:0:179192:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered
00010000:00000001:1.0:1788573025.046265:0:179192:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788573025.046266:0:179192:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff895343a49400 with handle 0xdb003797157261f6 from hash
00010000:00000001:1.0:1788573025.046268:0:179192:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving
00010000:00000001:1.0:1788573025.046269:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.046269:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046270:0:179192:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving
00010000:00000001:1.0:1788573025.046272:0:179192:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving
00010000:00000001:1.0:1788573025.046273:0:179192:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000)
00010000:00000001:1.0:1788573025.046274:0:179192:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered
00010000:00010000:1.0:1788573025.046276:0:179192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895351d53c00/0xdb00379715725d03 lrc: 2/0,0 mode: CR/CR res: [0x200002343:0x2319:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc27023d919f5308d expref: -99 pid: 210193 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0
00000080:00000001:1.0:1788573025.046280:0:179192:0:(namei.c:477:ll_md_blocking_ast()) Process entered
00000080:00000001:1.0:1788573025.046296:0:179192:0:(namei.c:240:ll_lock_cancel_bits()) Process entered
00000080:00000001:1.0:1788573025.046297:0:179192:0:(file.c:6096:ll_have_md_lock()) Process entered
00000080:00000040:1.0:1788573025.046298:0:179192:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002343:0x2319:0x0] mode (null)
00000002:00000001:1.0:1788573025.046301:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046302:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046304:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895379f7a300 count: 2
00010000:00000001:1.0:1788573025.046306:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046309:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046310:0:179192:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046312:0:179192:0:(file.c:6808:ll_layout_conf()) Process entered
00000020:00000001:1.0:1788573025.046313:0:179192:0:(cl_object.c:754:cl_env_obtain()) Process entered
00000020:00000001:1.0:1788573025.046314:0:179192:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446613590345126904 : -130483364424712 : ffff8953796a13f8)
00000020:00001000:1.0:1788573025.046316:0:179192:0:(cl_object.c:823:cl_env_get()) 1@ffff8953796a13f0
00000020:00000001:1.0:1788573025.046318:0:179192:0:(cl_object.c:343:cl_conf_set()) Process entered
00000080:00200000:1.0:1788573025.046320:0:179192:0:(vvp_object.c:117:vvp_conf_set()) [0x200002343:0x2319:0x0]: losing layout lock
00000080:00000001:1.0:1788573025.046322:0:179192:0:(pcc.c:2252:pcc_layout_invalidate()) Process entered
00000080:00000001:1.0:1788573025.046323:0:179192:0:(pcc.c:2264:pcc_layout_invalidate()) Process leaving
00020000:00000001:1.0:1788573025.046325:0:179192:0:(lov_object.c:1448:lov_conf_set()) Process entered
00020000:00000001:1.0:1788573025.046327:0:179192:0:(lov_object.c:1462:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0)
00020000:00000002:1.0:1788573025.046328:0:179192:0:(lov_object.c:1535:lov_conf_set()) [0x200002343:0x2319:0x0] lo_layout_invalid=1
00020000:00000001:1.0:1788573025.046329:0:179192:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573025.046331:0:179192:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00001000:1.0:1788573025.046332:0:179192:0:(cl_object.c:917:cl_env_put()) 1@ffff8953796a13f0
00000080:00000001:1.0:1788573025.046333:0:179192:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046335:0:179192:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving
00000080:00000001:1.0:1788573025.046336:0:179192:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1788573025.046337:0:179192:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered
00010000:00000001:1.0:1788573025.046338:0:179192:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered
00010000:00000001:1.0:1788573025.046340:0:179192:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered
00010000:00000001:1.0:1788573025.046341:0:179192:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered
00010000:00000001:1.0:1788573025.046343:0:179192:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788573025.046346:0:179192:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff895351d53c00 with handle 0xdb00379715725d03 from hash
00010000:00000001:1.0:1788573025.046348:0:179192:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving
00010000:00000001:1.0:1788573025.046350:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.046351:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046351:0:179192:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving
00010000:00000001:1.0:1788573025.046353:0:179192:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving
00010000:00000001:1.0:1788573025.046354:0:179192:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000)
00010000:00000001:1.0:1788573025.046355:0:179192:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered
00010000:00010000:1.0:1788573025.046357:0:179192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff89536d085400/0xdb00379715725cc4 lrc: 2/0,0 mode: PR/PR res: [0x200002343:0x2317:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xc27023d919f52f60 expref: -99 pid: 209582 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00000080:00000001:1.0:1788573025.046364:0:179192:0:(namei.c:477:ll_md_blocking_ast()) Process entered
00000080:00000001:1.0:1788573025.046366:0:179192:0:(namei.c:240:ll_lock_cancel_bits()) Process entered
00000080:00000001:1.0:1788573025.046367:0:179192:0:(file.c:6096:ll_have_md_lock()) Process entered
00000080:00000040:1.0:1788573025.046368:0:179192:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002343:0x2317:0x0] mode (null)
00000002:00000001:1.0:1788573025.046370:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046370:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046372:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895348471b00 count: 2
00010000:00000001:1.0:1788573025.046374:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046375:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.046376:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046377:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046378:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895348471b00 count: 2
00010000:00000001:1.0:1788573025.046379:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046381:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.046382:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046382:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046383:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895348471b00 count: 2
00010000:00000001:1.0:1788573025.046384:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046386:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000002:00000001:1.0:1788573025.046387:0:179192:0:(mdc_locks.c:137:mdc_lock_match()) Process entered
00010000:00000001:1.0:1788573025.046387:0:179192:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered
00010000:00000040:1.0:1788573025.046388:0:179192:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff895348471b00 count: 2
00010000:00000001:1.0:1788573025.046390:0:179192:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0)
00000002:00000001:1.0:1788573025.046434:0:179192:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046437:0:179192:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046438:0:179192:0:(file.c:6808:ll_layout_conf()) Process entered
00000020:00000001:1.0:1788573025.046440:0:179192:0:(cl_object.c:754:cl_env_obtain()) Process entered
00000020:00000001:1.0:1788573025.046442:0:179192:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446613590345126904 : -130483364424712 : ffff8953796a13f8)
00000020:00001000:1.0:1788573025.046444:0:179192:0:(cl_object.c:823:cl_env_get()) 1@ffff8953796a13f0
00000020:00000001:1.0:1788573025.046445:0:179192:0:(cl_object.c:343:cl_conf_set()) Process entered
00000080:00200000:1.0:1788573025.046447:0:179192:0:(vvp_object.c:117:vvp_conf_set()) [0x200002343:0x2317:0x0]: losing layout lock
00000080:00000001:1.0:1788573025.046448:0:179192:0:(pcc.c:2252:pcc_layout_invalidate()) Process entered
00000080:00000001:1.0:1788573025.046450:0:179192:0:(pcc.c:2264:pcc_layout_invalidate()) Process leaving
00020000:00000001:1.0:1788573025.046452:0:179192:0:(lov_object.c:1448:lov_conf_set()) Process entered
00020000:00000001:1.0:1788573025.046453:0:179192:0:(lov_object.c:1462:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0)
00020000:00000002:1.0:1788573025.046454:0:179192:0:(lov_object.c:1535:lov_conf_set()) [0x200002343:0x2317:0x0] lo_layout_invalid=1
00020000:00000001:1.0:1788573025.046456:0:179192:0:(lov_object.c:1536:lov_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573025.046458:0:179192:0:(cl_object.c:351:cl_conf_set()) Process leaving (rc=0 : 0 : 0)
00000020:00001000:1.0:1788573025.046459:0:179192:0:(cl_object.c:917:cl_env_put()) 1@ffff8953796a13f0
00000080:00000001:1.0:1788573025.046461:0:179192:0:(file.c:6850:ll_layout_conf()) Process leaving (rc=0 : 0 : 0)
00000080:00000001:1.0:1788573025.046464:0:179192:0:(dcache.c:195:ll_prune_aliases()) Process entered
00000080:00000002:1.0:1788573025.046466:0:179192:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002343:0x2317:0x0](ffff895351a75b88) invalid
00000020:00000001:1.0:1788573025.046469:0:179192:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff987001c47ca0
00000080:00002000:1.0:1788573025.046472:0:179192:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry f807b.sanity (ffff89535274c6c0) parent ffff89534219ca20 inode ffff895351a75b88 refc 0
00000080:00000001:1.0:1788573025.046477:0:179192:0:(dcache.c:39:ll_release()) Process entered
00000080:00000001:1.0:1788573025.046480:0:179192:0:(dcache.c:48:ll_release()) Process leaving
00000080:00000001:1.0:1788573025.046483:0:179192:0:(dcache.c:209:ll_prune_aliases()) Process leaving
00000080:00000001:1.0:1788573025.046484:0:179192:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving
00000080:00000001:1.0:1788573025.046485:0:179192:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1788573025.046487:0:179192:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered
00010000:00000001:1.0:1788573025.046488:0:179192:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered
00010000:00000001:1.0:1788573025.046489:0:179192:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered
00010000:00000001:1.0:1788573025.046490:0:179192:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered
00010000:00000001:1.0:1788573025.046491:0:179192:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788573025.046493:0:179192:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff89536d085400 with handle 0xdb00379715725cc4 from hash
00010000:00000001:1.0:1788573025.046495:0:179192:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving
00010000:00000001:1.0:1788573025.046495:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00000001:1.0:1788573025.046496:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046497:0:179192:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving
00010000:00000001:1.0:1788573025.046498:0:179192:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving
00010000:00000001:1.0:1788573025.046499:0:179192:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000)
00010000:00000001:1.0:1788573025.046501:0:179192:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=8 : 8 : 8)
00010000:00000001:1.0:1788573025.046504:0:179192:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered
00010000:00000001:1.0:1788573025.046506:0:179192:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered
00000100:00000010:1.0:1788573025.046513:0:179192:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5c00.
00000020:00000040:1.0:1788573025.046516:0:179192:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:1.0:1788573025.046520:0:179192:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573025.046523:0:179192:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573025.046526:0:179192:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573025.046533:0:179192:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff895349c3c000.
00000100:00000001:1.0:1788573025.046538:0:179192:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1788573025.046541:0:179192:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered
00010000:00010000:1.0:1788573025.046546:0:179192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895343a49600/0xdb00379715726274 lrc: 1/0,0 mode: --/CR res: [0x200002343:0x233a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f5426f expref: -99 pid: 215613 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0
00010000:00010000:1.0:1788573025.046551:0:179192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c00800/0xdb00379715726258 lrc: 1/0,0 mode: --/PR res: [0x200002343:0x2339:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f5421b expref: -99 pid: 213723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00010000:00010000:1.0:1788573025.046556:0:179192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c00e00/0xdb0037971572624a lrc: 1/0,0 mode: --/CR res: [0x200002343:0x2339:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f541f1 expref: -99 pid: 213721 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00010000:00010000:1.0:1788573025.046561:0:179192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c00c00/0xdb00379715726227 lrc: 1/0,0 mode: --/PR res: [0x200002343:0x2338:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f54196 expref: -99 pid: 212267 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0
00010000:00010000:1.0:1788573025.046565:0:179192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c01400/0xdb00379715726219 lrc: 1/0,0 mode: --/CR res: [0x200002343:0x2338:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f5416c expref: -99 pid: 212265 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00010000:00010000:1.0:1788573025.046569:0:179192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895343a49400/0xdb003797157261f6 lrc: 1/0,0 mode: --/PR res: [0x200002343:0x231d:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f540fc expref: -99 pid: 211547 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0
00010000:00010000:1.0:1788573025.046573:0:179192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895351d53c00/0xdb00379715725d03 lrc: 1/0,0 mode: --/CR res: [0x200002343:0x2319:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f5308d expref: -99 pid: 210193 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0
00010000:00010000:1.0:1788573025.046580:0:179192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff89536d085400/0xdb00379715725cc4 lrc: 1/0,0 mode: --/PR res: [0x200002343:0x2317:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f52f60 expref: -99 pid: 209582 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00010000:00010000:1.0:1788573025.046588:0:179192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 8 locks packed
00010000:00000001:1.0:1788573025.046589:0:179192:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=8 : 8 : 8)
00000100:00000001:1.0:1788573025.046593:0:179192:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573025.046597:0:179192:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573025.046606:0:179192:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000d797b21] to pc [ptlrpcd_00_00+0]  req@ffff8953796f5c00 x1875444393028352/t0(0) o103->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:17/18 lens 376/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295
00010000:00000001:1.0:1788573025.046631:0:179192:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0)
00010000:00000001:1.0:1788573025.046634:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00010000:1.0:1788573025.046637:0:179192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895343a49600/0xdb00379715726274 lrc: 0/0,0 mode: --/CR res: [0x200002343:0x233a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f5426f expref: -99 pid: 215613 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0
00010000:00000010:1.0:1788573025.046643:0:179192:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8953508a36c0.
00010000:00000040:1.0:1788573025.046649:0:179192:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff895342de0900 count: 0
00010000:00000010:1.0:1788573025.046653:0:179192:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff895372f5ad00.
00010000:00000001:1.0:1788573025.046658:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046659:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00010000:1.0:1788573025.046661:0:179192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c00800/0xdb00379715726258 lrc: 0/0,0 mode: --/PR res: [0x200002343:0x2339:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f5421b expref: -99 pid: 213723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00010000:00000010:1.0:1788573025.046668:0:179192:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff895372d9db40.
00010000:00000001:1.0:1788573025.046673:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046674:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00010000:1.0:1788573025.046676:0:179192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c00e00/0xdb0037971572624a lrc: 0/0,0 mode: --/CR res: [0x200002343:0x2339:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f541f1 expref: -99 pid: 213721 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00010000:00000010:1.0:1788573025.046681:0:179192:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff895372d9d4c0.
00010000:00000040:1.0:1788573025.046684:0:179192:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff895343c56100 count: 0
00010000:00000010:1.0:1788573025.046688:0:179192:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff89537959da80.
00010000:00000001:1.0:1788573025.046696:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046699:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00010000:1.0:1788573025.046705:0:179192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c00c00/0xdb00379715726227 lrc: 0/0,0 mode: --/PR res: [0x200002343:0x2338:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f54196 expref: -99 pid: 212267 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0
00010000:00000010:1.0:1788573025.046727:0:179192:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8953508a3880.
00010000:00000001:1.0:1788573025.046731:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046731:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00010000:1.0:1788573025.046733:0:179192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895372c01400/0xdb00379715726219 lrc: 0/0,0 mode: --/CR res: [0x200002343:0x2338:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f5416c expref: -99 pid: 212265 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00010000:00000010:1.0:1788573025.046740:0:179192:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8953508a3dc0.
00010000:00000040:1.0:1788573025.046743:0:179192:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff895343c56800 count: 0
00010000:00000010:1.0:1788573025.046749:0:179192:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff89537959d400.
00010000:00000001:1.0:1788573025.046756:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046759:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00010000:1.0:1788573025.046762:0:179192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895343a49400/0xdb003797157261f6 lrc: 0/0,0 mode: --/PR res: [0x200002343:0x231d:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f540fc expref: -99 pid: 211547 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0
00010000:00000010:1.0:1788573025.046780:0:179192:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff895372d9d240.
00010000:00000001:1.0:1788573025.046783:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046786:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00010000:1.0:1788573025.046788:0:179192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff895351d53c00/0xdb00379715725d03 lrc: 0/0,0 mode: --/CR res: [0x200002343:0x2319:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f5308d expref: -99 pid: 210193 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0
00010000:00000010:1.0:1788573025.046792:0:179192:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff89536d1d6280.
00010000:00000040:1.0:1788573025.046793:0:179192:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff895379f7a300 count: 0
00010000:00000010:1.0:1788573025.046796:0:179192:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff895372cf1080.
00010000:00000001:1.0:1788573025.046803:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046806:0:179192:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered
00010000:00010000:1.0:1788573025.046810:0:179192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff89524108d000 lock: ffff89536d085400/0xdb00379715725cc4 lrc: 0/0,0 mode: --/PR res: [0x200002343:0x2317:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xc27023d919f52f60 expref: -99 pid: 209582 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0
00010000:00000010:1.0:1788573025.046815:0:179192:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff895344d108c0.
00010000:00000040:1.0:1788573025.046819:0:179192:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff895348471b00 count: 0
00010000:00000010:1.0:1788573025.046823:0:179192:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff895372d5fd00.
00010000:00000001:1.0:1788573025.046830:0:179192:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving
00010000:00000001:1.0:1788573025.046835:0:179192:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000010:1.0:1788573025.046842:0:179192:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff895352b69180.
00010000:00000001:1.0:1788573025.046846:0:179192:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573025.046849:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573025.046851:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573025.046852:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573025.046853:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573025.046855:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573025.046860:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573025.046863:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573025.046866:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573025.046870:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000080:00000010:1.1:1788573025.051431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb440.
00000080:00000010:1.1:1788573025.051445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff89537c8cb200.
00000080:00000010:1.1:1788573025.055423:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8953654c5980.
00010000:00000010:1.1:1788573025.055435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895342de0900.
00010000:00000010:1.1:1788573025.055440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49600.
00010000:00000010:1.1:1788573025.055444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00800.
00010000:00000010:1.1:1788573025.055448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56100.
00010000:00000010:1.1:1788573025.055450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00e00.
00010000:00000010:1.1:1788573025.055452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c00c00.
00010000:00000010:1.1:1788573025.055454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895343c56800.
00010000:00000010:1.1:1788573025.055457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895372c01400.
00010000:00000010:1.1:1788573025.055460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895343a49400.
00010000:00000010:1.1:1788573025.055463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895379f7a300.
00010000:00000010:1.1:1788573025.055467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff895351d53c00.
00010000:00000010:1.1:1788573025.055469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff895348471b00.
00010000:00000010:1.1:1788573025.055471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff89536d085400.
00000800:00000200:0.1:1788573027.733994:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573027.733998:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573027.734376:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573027.734379:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573027.734722:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573027.734724:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 16 secs later)
00000100:00000040:1.0:1788573028.693470:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304632249, at 4304632256
00000100:00000010:1.0:1788573028.693481:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7840.
00000020:00000001:1.0:1788573029.141590:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573029.141593:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573029.141596:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573032.854281:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573032.854293:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788573032.854738:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573032.854739:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788573032.854965:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573032.854968:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000020:00000001:1.0:1788573033.237591:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573033.237596:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573033.237600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573033.813495:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304637370, at 4304637376
00000100:00000010:1.0:1788573033.813511:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000020:00000001:1.0:1788573037.333595:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573037.333599:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573037.333602:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573037.974165:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573037.974172:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788573037.974518:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573037.974519:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788573037.974759:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573037.974761:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00000040:1.0:1788573038.933486:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304642489, at 4304642496
00000100:00000010:1.0:1788573038.933503:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00000020:00000001:1.0:1788573041.429570:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573041.429575:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573041.429578:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573043.094010:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573043.094015:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788573043.094372:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573043.094373:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788573043.094634:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573043.094637:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000040:1.0:1788573044.053478:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304647609, at 4304647616
00000100:00000010:1.0:1788573044.053494:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000020:00000001:1.0:1788573045.525632:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573045.525647:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573045.525651:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573048.214016:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573048.214227:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573048.214334:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788573049.173494:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304652729, at 4304652736
00000100:00000010:1.0:1788573049.173511:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000020:00000001:1.0:1788573049.621636:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573049.621640:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573049.621645:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:1.0:1788573051.285639:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788573051.285673:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788573051.285681:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573051.285684:0:2351:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788573051.285689:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573051.285692:0:2351:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788573051.285696:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573051.285701:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:1.0:1788573051.285706:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573051.285711:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573051.285718:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65198
00000400:00000010:1.0:1788573051.285721:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65198.
00000100:00000001:1.0:1788573051.285726:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573051.285736:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e3800 x1875444393030016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573051.285760:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573051.285764:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393030016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573051.285769:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573051.285771:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573051.285781:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573051.285784:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:1.0:1788573051.285787:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573051.285791:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573051.285797:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:1.0:1788573051.285799:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:1.0:1788573051.285802:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573051.285807:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e1500 x1875444393030144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573051.285817:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573051.285821:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393030144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573051.285833:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573051.285837:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573051.285844:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573051.285850:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133c00.
00000400:00000200:1.0:1788573051.285856:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573051.285865:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573051.285869:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65bb0
00000400:00000010:1.0:1788573051.285874:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65bb0.
00000100:00000001:1.0:1788573051.285879:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573051.285886:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e1f80 x1875444393030272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573051.285903:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573051.285927:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393030272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573051.285945:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573051.285951:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000200:2.1:1788573053.334228:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573053.334235:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573053.334737:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573053.334740:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573053.335064:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573053.335067:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788573053.717578:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573053.717583:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573053.717587:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573054.293489:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304657849, at 4304657856
00000100:00000010:1.0:1788573054.293508:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000020:00000001:1.0:1788573057.813609:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573057.813613:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573057.813616:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573058.454055:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573058.454059:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573058.454488:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573058.454490:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573058.454752:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573058.454754:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:1.0:1788573059.413500:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304662969, at 4304662976
00000100:00000010:1.0:1788573059.413521:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000020:00000001:1.0:1788573061.909579:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573061.909582:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573061.909584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788573062.549663:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000200:2.1:1788573063.574091:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573063.574094:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573063.574464:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573063.574465:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573063.574668:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573063.574669:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:1.0:1788573064.533515:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304668089, at 4304668096
00000100:00000010:1.0:1788573064.533536:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000020:00000001:1.0:1788573066.005567:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573066.005571:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573066.005574:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573068.693512:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573068.693524:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573068.693527:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573068.693531:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573068.693536:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573068.693543:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 9999
00000400:00000200:1.0:1788573068.693553:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573068.693560:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573068.693563:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573068.693566:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693569:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573068.693571:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693573:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573068.693577:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573068.693578:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573068.693581:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573068.693584:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573068.693586:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573068.693588:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573068.693591:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693592:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573068.693594:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693595:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573068.693598:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573068.693601:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693602:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573068.693605:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573068.693608:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573068.693611:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573068.693612:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573068.693615:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573068.693625:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788573068.693628:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573068.693632:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573068.693634:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573068.693636:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573068.693639:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573068.693645:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573068.693653:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573068.693661:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573068.693673:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393031552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573068.693686:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573068.693692:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573068.693698:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393031552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573068.693724:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573068.693727:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573068.693730:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573068.693731:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573068.693733:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573068.693736:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573068.693738:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9999
00000400:00000200:1.0:1788573068.693741:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573068.693744:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573068.693746:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573068.693748:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693751:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573068.693752:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693754:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573068.693756:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573068.693756:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573068.693758:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573068.693760:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573068.693761:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573068.693763:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573068.693767:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693768:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573068.693770:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693772:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573068.693774:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573068.693777:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573068.693779:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573068.693780:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573068.693787:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573068.693789:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573068.693791:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573068.693801:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573068.693809:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573068.693818:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573068.693831:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788573068.693838:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573068.693849:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573068.693852:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573068.693859:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573068.693861:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573068.693867:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573068.693885:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573068.693895:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573068.693915:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393031680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573068.693926:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573068.693927:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573068.693933:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393031680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573068.693946:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573068.693957:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573068.693965:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573068.693967:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573068.693970:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573068.693981:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573068.693987:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 9999
00000400:00000200:1.0:1788573068.694001:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573068.694007:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573068.694009:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573068.694012:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.694018:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573068.694020:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.694023:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573068.694027:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573068.694030:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573068.694037:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573068.694040:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573068.694044:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573068.694049:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573068.694059:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.694062:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573068.694065:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573068.694075:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573068.694079:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573068.694085:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573068.694089:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573068.694095:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573068.694100:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573068.694103:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573068.694110:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573068.694115:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573068.694124:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573068.694127:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573068.694135:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788573068.694140:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573068.694143:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573068.694144:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573068.694146:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573068.694147:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573068.694149:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573068.694153:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573068.694158:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573068.694166:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393031808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573068.694174:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573068.694178:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573068.694184:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393031808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573068.694195:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573068.694198:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573068.694203:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573068.694225:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10009)
00000800:00000200:2.1:1788573068.694309:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573068.694313:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788573068.694714:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573068.694716:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788573068.694907:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573068.694909:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:1.0:1788573069.653479:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304673210, at 4304673216
00000100:00000010:1.0:1788573069.653499:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00000020:00000001:1.0:1788573070.101561:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573070.101565:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573070.101569:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573073.813991:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573073.813996:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788573073.814492:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573073.814495:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788573073.814699:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573073.814701:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:1.0:1788573074.197540:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573074.197544:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573074.197547:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573074.773504:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304678329, at 4304678336
00000100:00000010:1.0:1788573074.773519:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00000020:00000001:1.0:1788573078.293625:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573078.293631:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573078.293636:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573078.934298:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573078.934306:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788573078.934730:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573078.934731:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788573078.934964:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573078.934966:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:1.0:1788573079.445500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573079.445506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573079.445508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573079.445513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573079.445517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573079.445520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573079.445522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573079.445524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573079.445527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573079.893505:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304683450, at 4304683456
00000100:00000010:1.0:1788573079.893525:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000100:00000001:1.0:1788573080.469494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573080.469508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573080.469510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573080.469515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573080.469518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573080.469521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573080.469524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573080.469526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573080.469528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573081.493511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573081.493518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573081.493519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573081.493523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573081.493526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573081.493530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573081.493532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573081.493534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573081.493536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573082.389647:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573082.389653:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573082.389656:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573082.517505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573082.517514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573082.517516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573082.517522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573082.517526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573082.517529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573082.517532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573082.517535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573082.517537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573083.541507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573083.541514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573083.541516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573083.541521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573083.541523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573083.541527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573083.541529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573083.541530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573083.541532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573084.053964:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573084.053972:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788573084.054357:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573084.054362:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788573084.054582:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573084.054584:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1788573084.565477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573084.565483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573084.565485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573084.565489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573084.565492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573084.565495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573084.565497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573084.565499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573084.565501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573085.013463:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304688569, at 4304688576
00000100:00000010:1.0:1788573085.013479:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000100:00000001:1.0:1788573085.589496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573085.589502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573085.589505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573085.589510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573085.589512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573085.589515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573085.589517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573085.589520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573085.589523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573086.485642:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573086.485646:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573086.485648:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573086.613559:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573086.613571:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573086.613573:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573086.613579:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573086.613583:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573086.613586:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573086.613589:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573086.613592:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573086.613595:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573087.637599:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573087.637610:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573087.637614:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573087.637622:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573087.637627:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573087.637632:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573087.637636:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573087.637640:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573087.637644:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573088.661571:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573088.661580:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573088.661582:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573088.661589:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573088.661592:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573088.661596:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573088.661599:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573088.661601:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573088.661604:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573089.174214:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573089.174224:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788573089.174736:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573089.174738:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788573089.174964:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573089.174966:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1788573089.685564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573089.685573:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573089.685575:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573089.685580:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573089.685583:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573089.685588:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573089.685591:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573089.685594:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573089.685596:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573090.133550:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304693690, at 4304693696
00000100:00000010:1.0:1788573090.133573:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000020:00000001:1.0:1788573090.581660:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573090.581666:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573090.581670:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573090.709525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573090.709546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573090.709549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573090.709556:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573090.709559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573090.709563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573090.709566:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573090.709569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573090.709571:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573091.733564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573091.733576:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573091.733580:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573091.733587:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573091.733593:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573091.733598:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573091.733601:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573091.733605:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573091.733609:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573092.757503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573092.757510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573092.757513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573092.757525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573092.757530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573092.757535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573092.757538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573092.757542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573092.757545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573093.781473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573093.781479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573093.781481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573093.781485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573093.781487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573093.781491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573093.781493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573093.781495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573093.781497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573094.293522:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573094.293534:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573094.293537:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573094.293541:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573094.293546:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573094.293553:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10025
00000400:00000200:1.0:1788573094.293562:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573094.293568:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573094.293572:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573094.293575:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293580:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573094.293583:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293586:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573094.293590:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573094.293592:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573094.293595:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573094.293598:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573094.293601:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573094.293604:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573094.293607:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293609:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573094.293611:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293613:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573094.293617:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573094.293620:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293623:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573094.293627:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573094.293630:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573094.293634:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573094.293636:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573094.293638:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573094.293647:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788573094.293652:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573094.293656:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573094.293658:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573094.293661:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573094.293663:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573094.293667:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573094.293676:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788573094.293683:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573094.293706:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393033472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573094.293723:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573094.293728:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573094.293733:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393033472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573094.293756:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573094.293759:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573094.293763:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573094.293764:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573094.293767:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573094.293770:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573094.293773:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10025
00000400:00000200:1.0:1788573094.293778:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573094.293781:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573094.293783:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573094.293787:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293790:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573094.293792:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293795:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573094.293808:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573094.293813:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573094.293815:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573094.293820:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573094.293824:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573094.293828:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573094.293835:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293842:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573094.293847:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293861:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573094.293868:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573094.293872:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573094.293875:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573094.293879:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573094.293888:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573094.293894:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573094.293900:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573094.293906:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573094.293913:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573094.293919:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573094.293927:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788573094.293933:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573094.293941:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573094.293945:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573094.293950:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573094.293954:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573094.293957:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573094.293965:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573094.293972:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573094.293980:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393033600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573094.293995:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573094.294000:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573094.294010:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393033600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573094.294029:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573094.294032:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573094.294036:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573094.294042:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573094.294044:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573094.294050:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573094.294059:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10025
00000400:00000200:1.0:1788573094.294070:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573094.294078:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573094.294083:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573094.294091:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.294095:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573094.294102:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.294106:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573094.294111:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573094.294116:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573094.294121:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573094.294126:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573094.294131:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573094.294137:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573094.294146:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.294150:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573094.294156:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573094.294161:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573094.294167:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573094.294174:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573094.294179:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573094.294183:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573094.294188:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573094.294192:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573094.294196:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573094.294200:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573094.294205:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573094.294211:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573094.294215:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:1.0:1788573094.294217:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000800:00000200:2.1:1788573094.294220:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000040:1.0:1788573094.294222:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573094.294228:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573094.294231:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573094.294235:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573094.294239:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573094.294242:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573094.294280:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573094.294287:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573094.294298:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393033728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573094.294315:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573094.294320:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573094.294328:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393033728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573094.294350:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573094.294356:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573094.294363:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573094.294371:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10035)
00000800:00000200:2.1:1788573094.294828:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573094.294831:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788573094.295160:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573094.295163:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788573094.677566:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573094.677570:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573094.677573:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573094.805452:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573094.805459:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573094.805461:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573094.805465:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573094.805468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573094.805471:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573094.805473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573094.805475:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573094.805478:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573095.253473:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304698810, at 4304698816
00000100:00000010:1.0:1788573095.253489:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000100:00000001:1.0:1788573095.829469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573095.829474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573095.829476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573095.829481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573095.829484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573095.829487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573095.829489:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573095.829491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573095.829493:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573096.853550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573096.853564:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573096.853569:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573096.853578:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573096.853585:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573096.853593:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573096.853598:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573096.853604:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573096.853628:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573097.877513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573097.877522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573097.877525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573097.877530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573097.877534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573097.877538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573097.877540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573097.877543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573097.877546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573098.773619:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573098.773639:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573098.773641:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573098.901453:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573098.901459:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573098.901461:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573098.901465:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573098.901468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573098.901471:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573098.901473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573098.901475:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573098.901477:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573099.413489:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573099.413502:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573099.413506:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573099.413510:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573099.413515:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573099.413522:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10030
00000400:00000200:1.0:1788573099.413531:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573099.413538:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573099.413541:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573099.413545:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413549:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573099.413552:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413555:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573099.413559:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573099.413561:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573099.413564:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573099.413567:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573099.413570:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573099.413572:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573099.413576:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413578:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573099.413580:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413583:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573099.413586:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573099.413590:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413593:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573099.413596:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573099.413599:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573099.413617:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573099.413620:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573099.413622:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573099.413632:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788573099.413642:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573099.413645:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573099.413646:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573099.413649:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573099.413651:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573099.413654:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573099.413661:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573099.413666:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573099.413674:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393033856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573099.413682:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573099.413685:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573099.413689:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393033856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573099.413706:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573099.413709:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573099.413711:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573099.413713:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573099.413714:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573099.413716:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573099.413718:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10030
00000400:00000200:1.0:1788573099.413721:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573099.413723:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573099.413725:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573099.413727:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413728:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573099.413729:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413731:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573099.413732:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573099.413733:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573099.413735:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573099.413737:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573099.413739:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573099.413740:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573099.413743:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413745:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573099.413746:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413748:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573099.413749:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573099.413751:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413752:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573099.413754:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573099.413758:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573099.413762:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573099.413767:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573099.413775:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573099.413779:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573099.413781:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573099.413786:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788573099.413789:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573099.413794:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573099.413797:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573099.413802:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573099.413805:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573099.413809:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573099.413817:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573099.413825:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573099.413837:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393033984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573099.413851:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573099.413856:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573099.413865:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393033984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573099.413902:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573099.413907:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573099.413915:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573099.413921:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573099.413924:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573099.413931:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573099.413937:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10030
00000400:00000200:1.0:1788573099.413947:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573099.413954:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573099.413959:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573099.413966:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413981:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573099.413988:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.413995:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573099.414000:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573099.414003:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573099.414007:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573099.414012:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573099.414017:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573099.414022:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573099.414029:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.414035:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573099.414040:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573099.414043:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573099.414048:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:2.1:1788573099.414064:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:1.0:1788573099.414069:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:2.1:1788573099.414071:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788573099.414073:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573099.414078:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573099.414083:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573099.414088:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573099.414090:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573099.414101:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573099.414108:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573099.414113:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573099.414119:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788573099.414124:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573099.414129:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573099.414133:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573099.414137:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573099.414140:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573099.414144:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573099.414150:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788573099.414157:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573099.414167:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393034112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573099.414198:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573099.414203:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573099.414210:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393034112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573099.414224:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573099.414228:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573099.414232:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573099.414240:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10040)
00000800:00000200:2.1:1788573099.414565:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573099.414568:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788573099.414888:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573099.414890:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788573099.925477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573099.925484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573099.925486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573099.925491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573099.925495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573099.925499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573099.925502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573099.925504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573099.925507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573100.373481:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304703929, at 4304703936
00000100:00000010:1.0:1788573100.373498:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000800:00000200:1.0:1788573100.437705:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000001:1.0:1788573100.949538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573100.949545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573100.949547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573100.949553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573100.949556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573100.949560:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573100.949562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573100.949580:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573100.949582:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573101.973519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573101.973527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573101.973529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573101.973534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573101.973538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573101.973542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573101.973545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573101.973548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573101.973550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573102.869529:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573102.869531:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573102.869533:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573102.997493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573102.997499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573102.997500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573102.997504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573102.997507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573102.997511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573102.997513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573102.997515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573102.997517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573104.021482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573104.021494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573104.021497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573104.021505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573104.021510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573104.021515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573104.021519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573104.021522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573104.021526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573104.533509:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573104.533521:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573104.533524:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573104.533536:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573104.533540:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573104.533546:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10035
00000400:00000200:1.0:1788573104.533555:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573104.533561:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573104.533564:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573104.533566:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533568:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573104.533571:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533572:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573104.533576:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573104.533577:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573104.533579:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573104.533582:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573104.533584:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573104.533585:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573104.533587:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533589:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573104.533590:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533592:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573104.533594:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573104.533597:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533599:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573104.533601:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573104.533604:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573104.533606:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573104.533608:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573104.533609:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573104.533617:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788573104.533620:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573104.533622:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573104.533624:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573104.533626:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573104.533629:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573104.533632:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573104.533638:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573104.533644:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573104.533651:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393034240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573104.533659:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573104.533662:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573104.533665:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393034240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573104.533684:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573104.533686:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573104.533690:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573104.533692:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573104.533694:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573104.533697:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573104.533699:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10035
00000400:00000200:1.0:1788573104.533704:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573104.533708:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573104.533711:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573104.533716:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533719:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573104.533721:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533724:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573104.533730:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573104.533734:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573104.533738:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573104.533742:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573104.533748:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573104.533751:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573104.533755:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533757:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573104.533763:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533768:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573104.533774:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573104.533780:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573104.533783:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573104.533786:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573104.533798:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573104.533804:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573104.533808:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573104.533812:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573104.533816:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573104.533820:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573104.533825:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788573104.533830:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573104.533836:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573104.533841:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573104.533844:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573104.533847:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573104.533850:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573104.533870:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788573104.533879:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573104.533888:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393034368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573104.533913:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573104.533919:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573104.533929:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393034368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573104.533945:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573104.533949:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573104.533953:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573104.533956:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573104.533960:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573104.533963:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573104.533968:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10035
00000400:00000200:1.0:1788573104.533974:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573104.533981:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573104.533985:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573104.533998:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.534002:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573104.534007:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.534009:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573104.534014:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573104.534016:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573104.534019:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573104.534023:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573104.534025:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573104.534027:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573104.534032:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.534037:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573104.534044:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573104.534050:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.1:1788573104.534054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:1.0:1788573104.534056:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:2.1:1788573104.534058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00080000:1.0:1788573104.534064:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573104.534071:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573104.534076:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573104.534082:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573104.534087:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573104.534092:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573104.534096:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573104.534099:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573104.534101:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573104.534107:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788573104.534112:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573104.534116:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573104.534118:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573104.534120:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573104.534123:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573104.534126:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573104.534132:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788573104.534137:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573104.534146:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393034496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573104.534153:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573104.534157:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573104.534162:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393034496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573104.534171:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573104.534175:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573104.534178:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573104.534184:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10045)
00000800:00000200:2.1:1788573104.534495:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573104.534496:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788573104.534727:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573104.534729:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:1.0:1788573105.045503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573105.045510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573105.045512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573105.045518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573105.045523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573105.045527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573105.045530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573105.045533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573105.045536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573105.493520:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304709049, at 4304709056
00000100:00000010:1.0:1788573105.493536:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000100:00000001:1.0:1788573106.069509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573106.069518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573106.069521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573106.069527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573106.069531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573106.069535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573106.069538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573106.069541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573106.069544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573106.965689:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573106.965694:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573106.965699:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573107.093541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573107.093550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573107.093552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573107.093558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573107.093562:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573107.093566:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573107.093568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573107.093571:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573107.093573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573108.117483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573108.117489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573108.117491:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573108.117495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573108.117498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573108.117502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573108.117504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573108.117506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573108.117508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573109.141483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573109.141489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573109.141491:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573109.141496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573109.141499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573109.141502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573109.141504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573109.141506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573109.141508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573109.654034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573109.654038:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788573109.654447:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573109.654449:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788573109.654700:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573109.654702:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:1.0:1788573110.165498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573110.165506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573110.165508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573110.165514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573110.165519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573110.165522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573110.165525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573110.165528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573110.165531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573110.613507:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304714169, at 4304714176
00000100:00000010:1.0:1788573110.613526:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00000020:00000001:1.0:1788573111.061577:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573111.061581:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573111.061584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573111.189501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573111.189510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573111.189512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573111.189517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573111.189521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573111.189525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573111.189527:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573111.189530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573111.189533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573112.213541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573112.213571:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573112.213575:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573112.213583:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573112.213588:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573112.213593:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573112.213597:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573112.213601:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573112.213604:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573113.237541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573113.237550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573113.237552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573113.237558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573113.237562:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573113.237566:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573113.237568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573113.237572:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573113.237575:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573114.261529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573114.261537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573114.261539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573114.261547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573114.261551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573114.261554:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573114.261557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573114.261560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573114.261570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573114.773978:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573114.774259:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573114.774366:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788573115.157606:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573115.157609:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573115.157612:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573115.285482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573115.285488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573115.285489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573115.285494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573115.285496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573115.285499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573115.285501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573115.285503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573115.285506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573115.733491:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304719289, at 4304719296
00000100:00000010:1.0:1788573115.733505:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000100:00000001:1.0:1788573116.309505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573116.309513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573116.309515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573116.309521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573116.309524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573116.309528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573116.309531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573116.309534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573116.309537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573117.333493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573117.333499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573117.333501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573117.333506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573117.333509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573117.333511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573117.333514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573117.333516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573117.333519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573118.357459:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573118.357466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573118.357468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573118.357471:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573118.357474:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573118.357478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573118.357481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573118.357484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573118.357487:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573119.253544:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573119.253547:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573119.253550:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573119.381487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573119.381494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573119.381496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573119.381500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573119.381503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573119.381507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573119.381509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573119.381511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573119.381513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573119.894091:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573119.894097:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573119.894804:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573119.894807:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573119.895094:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573119.895096:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:1.0:1788573120.405523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573120.405531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573120.405534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573120.405539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573120.405543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573120.405547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573120.405550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573120.405553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573120.405556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573120.853493:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304724409, at 4304724416
00000100:00000010:1.0:1788573120.853509:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d80.
00000100:00000001:1.0:1788573121.429497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573121.429506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573121.429509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573121.429516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573121.429520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573121.429525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573121.429528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573121.429530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573121.429533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573122.453489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573122.453495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573122.453496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573122.453500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573122.453503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573122.453507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573122.453509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573122.453511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573122.453513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573123.349576:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573123.349581:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573123.349584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573123.477469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573123.477476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573123.477478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573123.477482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573123.477485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573123.477488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573123.477490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573123.477492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573123.477494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573124.501505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573124.501510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573124.501512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573124.501516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573124.501519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573124.501521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573124.501523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573124.501525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573124.501527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573125.014091:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573125.014093:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573125.014473:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573125.014475:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573125.014693:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573125.014695:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788573125.525494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573125.525501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573125.525503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573125.525509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573125.525512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573125.525516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573125.525519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573125.525521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573125.525524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573125.973502:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304729529, at 4304729536
00000100:00000010:1.0:1788573125.973530:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7660.
00000100:00000001:1.0:1788573126.549491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573126.549499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573126.549501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573126.549507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573126.549511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573126.549515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573126.549518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573126.549521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573126.549523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573127.445612:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573127.445616:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573127.445619:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573127.573495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573127.573501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573127.573502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573127.573506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573127.573509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573127.573513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573127.573515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573127.573517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573127.573519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573128.597490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573128.597496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573128.597498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573128.597503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573128.597506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573128.597509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573128.597511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573128.597513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573128.597516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573129.621505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573129.621512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573129.621514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573129.621519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573129.621523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573129.621527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573129.621530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573129.621533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573129.621536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573130.134073:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573130.134082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573130.134453:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573130.134455:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573130.134694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573130.134696:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:1.0:1788573130.645535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573130.645542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573130.645543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573130.645547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573130.645550:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573130.645554:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573130.645556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573130.645557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573130.645560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573131.093493:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304734649, at 4304734656
00000100:00000010:1.0:1788573131.093511:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d70c0.
00000020:00000001:1.0:1788573131.541589:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573131.541592:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573131.541595:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573131.669500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573131.669507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573131.669509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573131.669513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573131.669516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573131.669520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573131.669522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573131.669524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573131.669526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573132.693497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573132.693504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573132.693506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573132.693510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573132.693512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573132.693516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573132.693517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573132.693519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573132.693521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573133.717468:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573133.717474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573133.717476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573133.717481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573133.717484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573133.717486:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573133.717488:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573133.717490:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573133.717492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573134.741448:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573134.741453:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573134.741455:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573134.741460:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573134.741463:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573134.741465:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573134.741467:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573134.741469:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573134.741471:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573135.254032:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573135.254037:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788573135.254440:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573135.254442:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788573135.254656:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573135.254658:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:1.0:1788573135.637603:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573135.637606:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573135.637609:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573135.765499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573135.765508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573135.765512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573135.765518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573135.765522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573135.765527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573135.765530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573135.765533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573135.765537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573136.213450:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304739769, at 4304739776
00000100:00000010:1.0:1788573136.213464:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ae0.
00000100:00000001:1.0:1788573136.789496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573136.789505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573136.789507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573136.789514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573136.789517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573136.789520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573136.789523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573136.789526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573136.789529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573137.813457:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573137.813464:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573137.813465:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573137.813470:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573137.813472:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573137.813475:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573137.813477:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573137.813479:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573137.813481:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788573138.325497:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788573138.325507:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788573138.325512:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788573138.325709:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573138.325725:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573138.325732:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573138.325736:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573138.837486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573138.837494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573138.837496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573138.837502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573138.837506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573138.837509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573138.837512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573138.837514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573138.837516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573139.733603:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573139.733609:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573139.733613:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573139.861455:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573139.861461:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573139.861463:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573139.861468:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573139.861471:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573139.861473:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573139.861475:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573139.861477:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573139.861479:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573140.374060:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573140.374064:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788573140.374407:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573140.374410:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788573140.374694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573140.374697:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:1.0:1788573140.885476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573140.885498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573140.885500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573140.885504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573140.885507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573140.885510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573140.885512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573140.885514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573140.885516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573141.333507:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304744889, at 4304744896
00000100:00000010:1.0:1788573141.333522:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d79c0.
00000100:00000001:1.0:1788573141.909550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573141.909560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573141.909563:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573141.909568:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573141.909572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573141.909576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573141.909579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573141.909582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573141.909585:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573142.933477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573142.933484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573142.933486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573142.933491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573142.933495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573142.933498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573142.933500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573142.933503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573142.933506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573143.829566:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573143.829572:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573143.829577:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573143.957571:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573143.957582:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573143.957585:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573143.957593:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573143.957598:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573143.957604:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573143.957608:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573143.957612:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573143.957616:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573144.981490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573144.981499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573144.981501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573144.981507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573144.981510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573144.981515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573144.981518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573144.981521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573144.981524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573145.494255:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573145.494263:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788573145.494787:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573145.494789:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788573145.494978:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573145.494980:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:1.0:1788573146.005529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573146.005535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573146.005537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573146.005541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573146.005544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573146.005547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573146.005549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573146.005551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573146.005553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573146.453503:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304750010, at 4304750016
00000100:00000010:1.0:1788573146.453519:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7360.
00000100:00000001:1.0:1788573147.029592:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573147.029601:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573147.029604:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573147.029609:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573147.029613:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573147.029617:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573147.029620:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573147.029623:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573147.029625:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573147.925649:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573147.925654:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573147.925657:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573148.053517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573148.053526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573148.053528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573148.053533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573148.053537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573148.053541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573148.053544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573148.053547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573148.053549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573149.077508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573149.077518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573149.077520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573149.077526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573149.077529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573149.077533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573149.077535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573149.077537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573149.077539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573150.101494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573150.101502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573150.101505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573150.101513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573150.101516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573150.101520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573150.101523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573150.101526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573150.101528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573150.614032:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573150.614035:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1788573150.614477:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573150.614480:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1788573150.614691:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573150.614693:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00000001:1.0:1788573151.125462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573151.125469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573151.125471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573151.125475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573151.125478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573151.125480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573151.125482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573151.125484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573151.125487:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573151.573477:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304755129, at 4304755136
00000100:00000010:1.0:1788573151.573489:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d72a0.
00000020:00000001:1.0:1788573152.021602:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573152.021607:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573152.021611:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573152.149484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573152.149490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573152.149492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573152.149496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573152.149499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573152.149502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573152.149505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573152.149507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573152.149509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573155.734038:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573155.734043:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1788573155.734359:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573155.734361:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1788573155.734587:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573155.734590:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788573156.117606:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573156.117610:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573156.117612:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573156.693474:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304760249, at 4304760256
00000100:00000010:1.0:1788573156.693487:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7420.
00000020:00000001:1.0:1788573160.213609:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573160.213613:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573160.213616:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573160.854033:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573160.854038:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573160.854374:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573160.854376:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573160.854623:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573160.854625:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:1.0:1788573161.813497:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304765369, at 4304765376
00000100:00000010:1.0:1788573161.813506:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d74e0.
00000020:00000001:1.0:1788573164.309591:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573164.309596:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573164.309600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573164.437539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573164.437545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573164.437547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573164.437552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573164.437562:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573164.437565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573164.437568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573164.437570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573164.437573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573165.461499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573165.461506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573165.461508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573165.461512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573165.461515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573165.461518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573165.461520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573165.461522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573165.461524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573165.974103:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573165.974107:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573165.974665:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573165.974669:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573165.974987:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573165.974990:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788573166.485457:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573166.485464:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573166.485466:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573166.485470:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573166.485473:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573166.485476:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573166.485477:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573166.485479:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573166.485481:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573166.933499:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304770489, at 4304770496
00000100:00000010:1.0:1788573166.933534:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7000.
00000100:00000001:1.0:1788573167.509623:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573167.509631:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573167.509634:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573167.509642:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573167.509648:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573167.509654:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573167.509659:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573167.509663:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573167.509668:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573168.405605:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573168.405609:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573168.405612:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573168.533501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573168.533508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573168.533510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573168.533515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573168.533550:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573168.533553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573168.533555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573168.533557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573168.533560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573171.094040:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573171.094043:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573171.094544:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573171.094547:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573171.094867:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573171.094869:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:1.0:1788573172.053498:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304775609, at 4304775616
00000100:00000010:1.0:1788573172.053516:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7000.
00000020:00000001:1.0:1788573172.501564:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573172.501570:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573172.501573:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573176.214044:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573176.214049:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788573176.214547:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573176.214549:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788573176.214769:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573176.214771:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:1.0:1788573176.597580:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573176.597585:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573176.597589:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573177.173458:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304780729, at 4304780736
00000100:00000010:1.0:1788573177.173478:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d74e0.
00000800:00000001:1.0:1788573177.237699:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573177.237714:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573177.237719:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573177.237721:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788573180.693534:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573180.693537:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573180.693539:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573181.333494:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573181.333502:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573181.333504:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573181.333508:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573181.333511:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573181.333517:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10112
00000400:00000200:1.0:1788573181.333524:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573181.333529:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573181.333531:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573181.333534:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333536:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573181.333538:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333540:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573181.333543:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573181.333544:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573181.333546:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573181.333549:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573181.333551:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573181.333552:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573181.333554:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333555:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573181.333557:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333559:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573181.333561:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573181.333563:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333566:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573181.333568:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573181.333570:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573181.333572:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573181.333574:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573181.333576:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573181.333583:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788573181.333585:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573181.333588:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573181.333589:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573181.333592:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573181.333594:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573181.333597:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573181.333603:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573181.333610:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573181.333620:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393040000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573181.333632:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573181.333636:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573181.333641:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393040000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573181.333669:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573181.333680:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573181.333683:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573181.333685:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573181.333687:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573181.333690:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573181.333693:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10112
00000400:00000200:1.0:1788573181.333698:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573181.333702:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573181.333705:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573181.333708:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333710:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573181.333713:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333715:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573181.333718:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573181.333722:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573181.333726:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573181.333730:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573181.333735:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573181.333739:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573181.333744:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333750:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573181.333754:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333759:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573181.333764:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573181.333769:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573181.333774:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573181.333778:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573181.333787:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573181.333793:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573181.333798:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573181.333804:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573181.333810:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573181.333822:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573181.333828:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788573181.333831:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573181.333835:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573181.333840:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573181.333841:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573181.333843:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573181.333847:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573181.333854:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788573181.333860:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573181.333869:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393040128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573181.333886:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573181.333902:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573181.333910:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393040128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573181.333929:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573181.333936:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573181.333942:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573181.333944:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573181.333947:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573181.333952:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573181.333957:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10112
00000400:00000200:1.0:1788573181.334005:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573181.334010:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573181.334012:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573181.334018:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.334020:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573181.334028:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.334033:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573181.334036:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573181.334039:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573181.334044:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:0.1:1788573181.334044:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:1.0:1788573181.334048:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573181.334052:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573181.334056:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573181.334061:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.334064:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573181.334066:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573181.334087:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573181.334093:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573181.334099:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573181.334103:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573181.334105:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573181.334108:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573181.334118:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573181.334120:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573181.334123:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573181.334125:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573181.334129:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573181.334134:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788573181.334140:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573181.334147:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573181.334149:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573181.334155:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573181.334158:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573181.334161:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573181.334167:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788573181.334172:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573181.334181:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393040256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573181.334195:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573181.334198:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573181.334211:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393040256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573181.334239:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573181.334243:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573181.334255:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573181.334264:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10122)
00000800:00000200:0.1:1788573181.334377:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573181.334590:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788573182.293510:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304785849, at 4304785856
00000100:00000010:1.0:1788573182.293528:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7420.
00000400:00000100:1.0:1788573184.405785:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788573184.405841:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788573184.405851:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573184.405855:0:2352:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788573184.405860:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573184.405863:0:2352:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788573184.405868:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573184.405872:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:1.0:1788573184.405880:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573184.405885:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573184.405892:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4bb0
00000400:00000010:1.0:1788573184.405895:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4bb0.
00000100:00000001:1.0:1788573184.405900:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573184.405912:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f7b80 x1875444393040000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573184.405952:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573184.405959:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393040000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573184.405985:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573184.405987:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573184.406011:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573184.406013:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9300.
00000400:00000200:1.0:1788573184.406017:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573184.406020:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573184.406024:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:1.0:1788573184.406026:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:1.0:1788573184.406029:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573184.406033:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f4e00 x1875444393040128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573184.406044:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573184.406047:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393040128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573184.406053:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573184.406054:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573184.406059:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573184.406061:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:1.0:1788573184.406064:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573184.406067:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573184.406076:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4a18
00000400:00000010:1.0:1788573184.406078:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4a18.
00000100:00000001:1.0:1788573184.406080:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573184.406084:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f5500 x1875444393040256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573184.406092:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573184.406095:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5500 x1875444393040256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573184.406101:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573184.406103:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000001:1.0:1788573184.789582:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573184.789586:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573184.789589:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573186.454240:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573186.454244:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573186.454756:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573186.454759:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573186.455088:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573186.455091:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:1.0:1788573187.413488:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304790969, at 4304790976
00000100:00000010:1.0:1788573187.413506:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d72a0.
00000020:00000001:1.0:1788573188.885599:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573188.885603:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573188.885606:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573191.574128:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573191.574131:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573191.575916:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573191.575918:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573191.576225:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573191.576227:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:1.0:1788573192.533491:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304796089, at 4304796096
00000100:00000010:1.0:1788573192.533516:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7360.
00000020:00000001:1.0:1788573192.981623:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573192.981628:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573192.981632:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573195.029626:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573195.029632:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573195.029635:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573196.053590:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573196.053600:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573196.053604:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573196.694106:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573196.694110:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573196.694488:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573196.694491:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573196.694728:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573196.694731:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:1.0:1788573197.653527:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304801209, at 4304801216
00000100:00000010:1.0:1788573197.653540:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d79c0.
00000020:00000001:1.0:1788573198.101594:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573198.101602:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573198.101607:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573199.125607:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573199.125611:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573199.125614:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573201.173676:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573201.173681:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573201.173686:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573201.813569:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573201.813585:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573201.813588:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573201.813593:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573201.813599:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573201.813608:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10132
00000400:00000200:1.0:1788573201.813621:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573201.813629:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573201.813632:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573201.813637:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813640:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573201.813643:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813648:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573201.813652:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573201.813655:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573201.813659:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573201.813663:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573201.813666:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573201.813668:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573201.813672:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813674:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573201.813677:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813679:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573201.813683:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573201.813687:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813691:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573201.813694:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573201.813699:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573201.813703:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573201.813706:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573201.813709:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573201.813723:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788573201.813728:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573201.813733:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573201.813735:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573201.813740:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573201.813743:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573201.813750:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573201.813759:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788573201.813768:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573201.813780:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393041536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573201.813793:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573201.813799:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573201.813805:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393041536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573201.813830:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573201.813834:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573201.813838:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573201.813839:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573201.813841:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573201.813845:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573201.813848:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10132
00000400:00000200:1.0:1788573201.813852:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573201.813857:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573201.813860:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573201.813864:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813868:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573201.813874:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813886:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573201.813892:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573201.813894:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573201.813896:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573201.813901:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573201.813906:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573201.813912:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573201.813920:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813926:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573201.813931:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813935:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573201.813940:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573201.813948:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573201.813964:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573201.813970:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573201.813978:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573201.813984:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573201.813986:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573201.813990:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573201.813996:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573201.813999:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573201.814005:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788573201.814009:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573201.814013:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573201.814014:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573201.814016:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573201.814018:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573201.814022:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573201.814028:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573201.814034:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573201.814044:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393041664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573201.814074:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573201.814078:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573201.814085:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393041664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573201.814101:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573201.814103:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573201.814109:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573201.814112:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573201.814113:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573201.814117:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573201.814122:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10132
00000400:00000200:1.0:1788573201.814155:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573201.814163:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573201.814167:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573201.814172:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.814176:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573201.814183:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.814186:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573201.814189:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573201.814192:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:0.1:1788573201.814193:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788573201.814194:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573201.814197:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:0.1:1788573201.814198:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:1.0:1788573201.814200:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573201.814204:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573201.814210:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.814213:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573201.814215:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573201.814216:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573201.814220:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573201.814225:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573201.814229:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573201.814231:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573201.814234:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573201.814239:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573201.814240:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573201.814244:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573201.814246:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573201.814250:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573201.814255:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788573201.814260:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573201.814265:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573201.814268:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573201.814271:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573201.814272:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573201.814274:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573201.814279:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573201.814284:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573201.814293:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393041792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573201.814305:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573201.814308:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573201.814314:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393041792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573201.814324:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573201.814328:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573201.814332:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573201.814336:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10143)
00000800:00000200:0.1:1788573201.814766:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573201.814770:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788573201.815022:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573201.815024:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:1.0:1788573202.197634:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573202.197640:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573202.197644:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573202.773513:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304806329, at 4304806336
00000100:00000010:1.0:1788573202.773531:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ae0.
00000020:00000001:1.0:1788573204.245576:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573204.245580:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573204.245583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573205.269598:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573205.269602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573205.269605:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573206.933479:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573206.933491:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573206.933495:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573206.933499:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573206.933504:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573206.933512:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10138
00000400:00000200:1.0:1788573206.933520:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573206.933527:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573206.933531:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573206.933535:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933539:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573206.933541:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933544:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573206.933548:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573206.933550:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573206.933553:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573206.933557:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573206.933559:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573206.933562:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573206.933566:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933568:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573206.933570:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933573:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573206.933576:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573206.933579:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933582:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573206.933586:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573206.933589:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573206.933593:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573206.933595:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573206.933598:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573206.933607:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788573206.933611:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573206.933615:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573206.933618:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573206.933621:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573206.933624:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573206.933629:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573206.933636:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573206.933643:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573206.933653:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393041920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573206.933665:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573206.933669:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573206.933674:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393041920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573206.933697:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573206.933701:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573206.933705:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573206.933707:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573206.933708:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573206.933712:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573206.933716:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10138
00000400:00000200:1.0:1788573206.933721:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573206.933727:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573206.933732:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573206.933740:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933744:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573206.933750:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933755:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573206.933761:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573206.933765:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573206.933769:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573206.933774:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573206.933782:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573206.933789:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573206.933797:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933803:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573206.933807:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933812:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573206.933815:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573206.933819:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573206.933826:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573206.933831:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573206.933838:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573206.933842:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573206.933845:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573206.933852:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573206.933857:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573206.933860:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573206.933867:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788573206.933874:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573206.933880:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573206.933883:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573206.933887:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573206.933891:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573206.933896:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573206.933903:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573206.933909:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573206.933919:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393042048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573206.933933:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573206.933938:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573206.933946:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393042048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573206.933966:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573206.933973:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573206.933983:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573206.933988:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573206.933991:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573206.934000:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573206.934007:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10138
00000400:00000200:1.0:1788573206.934016:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573206.934023:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573206.934026:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573206.934033:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.934040:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573206.934043:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.934049:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573206.934053:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.1:1788573206.934055:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:1.0:1788573206.934056:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573206.934060:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:0.1:1788573206.934062:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000040:1.0:1788573206.934063:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573206.934069:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573206.934075:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573206.934082:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.934085:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573206.934089:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573206.934095:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573206.934101:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573206.934107:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573206.934113:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573206.934117:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573206.934121:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573206.934126:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573206.934132:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573206.934136:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573206.934141:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573206.934146:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573206.934154:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788573206.934161:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573206.934166:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573206.934168:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573206.934173:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573206.934178:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573206.934183:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573206.934191:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788573206.934196:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573206.934207:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393042176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573206.934225:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573206.934231:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573206.934250:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393042176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573206.934267:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573206.934274:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573206.934281:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573206.934289:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10148)
00000800:00000200:0.1:1788573206.934571:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573206.934573:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788573206.934790:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573206.934793:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:1.0:1788573207.317606:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573207.317611:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573207.317615:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573207.893475:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304811449, at 4304811456
00000100:00000010:1.0:1788573207.893492:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d70c0.
00000020:00000001:1.0:1788573208.341537:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573208.341540:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573208.341543:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573210.389536:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573210.389540:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573210.389550:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573211.413587:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573211.413590:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573211.413594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573212.053504:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573212.053515:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573212.053518:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573212.053522:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573212.053526:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573212.053535:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10143
00000400:00000200:1.0:1788573212.053545:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573212.053554:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573212.053557:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573212.053561:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053565:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573212.053568:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053571:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573212.053576:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573212.053579:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573212.053583:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573212.053588:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573212.053591:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573212.053593:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573212.053595:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053597:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573212.053599:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053600:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573212.053602:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573212.053605:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053607:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573212.053609:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573212.053613:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573212.053616:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573212.053618:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573212.053621:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573212.053631:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788573212.053634:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573212.053638:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573212.053640:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573212.053642:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573212.053645:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573212.053649:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573212.053657:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573212.053664:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573212.053673:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393042304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573212.053682:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573212.053686:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573212.053689:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393042304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573212.053714:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573212.053718:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573212.053720:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573212.053722:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573212.053723:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573212.053726:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573212.053727:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10143
00000400:00000200:1.0:1788573212.053730:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573212.053733:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573212.053735:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573212.053737:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053739:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573212.053740:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053742:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573212.053743:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573212.053745:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573212.053747:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573212.053749:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573212.053751:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573212.053753:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573212.053756:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053757:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573212.053758:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053760:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573212.053762:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573212.053763:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573212.053766:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573212.053768:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573212.053775:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573212.053777:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573212.053779:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573212.053783:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573212.053788:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573212.053790:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573212.053795:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788573212.053799:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573212.053803:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573212.053806:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573212.053810:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573212.053811:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573212.053815:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573212.053820:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573212.053824:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573212.053831:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393042432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573212.053842:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573212.053845:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573212.053969:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393042432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573212.053997:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573212.054004:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573212.054011:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573212.054015:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573212.054019:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573212.054026:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573212.054036:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10143
00000400:00000200:1.0:1788573212.054078:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573212.054088:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573212.054096:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573212.054105:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.054111:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573212.054121:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.054126:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573212.054134:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573212.054138:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:0.1:1788573212.054140:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788573212.054143:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:0.1:1788573212.054153:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000040:1.0:1788573212.054154:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573212.054163:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573212.054173:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573212.054186:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.054194:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573212.054198:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573212.054235:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573212.054241:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573212.054249:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573212.054258:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573212.054263:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573212.054272:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573212.054285:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573212.054292:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573212.054297:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573212.054302:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573212.054306:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573212.054320:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788573212.054327:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573212.054337:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573212.054340:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573212.054344:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573212.054349:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573212.054353:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573212.054362:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573212.054371:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573212.054384:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393042560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573212.054444:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573212.054451:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573212.054467:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393042560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573212.054486:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573212.054496:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573212.054509:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573212.054515:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10153)
00000800:00000200:0.1:1788573212.054872:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573212.054875:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573212.055225:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573212.055229:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:1.0:1788573212.565598:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573212.565606:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573212.565609:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573212.565615:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573212.565620:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573212.565623:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573212.565627:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573212.565629:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573212.565633:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573213.013521:0:206940:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304816570, at 4304816576
00000100:00000010:1.0:1788573213.013551:0:206940:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7660.
00000020:00000001:1.0:1788573213.461569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573213.461579:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573213.461582:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573213.589454:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573213.589461:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573213.589463:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573213.589466:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573213.589469:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573213.589473:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573213.589475:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573213.589477:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573213.589479:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573214.485576:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573214.485581:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573214.485584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788573215.125693:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573216.533574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573216.533577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573216.533580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573217.174038:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573217.174051:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573217.174426:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573217.174428:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573217.174617:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573217.174619:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:1.0:1788573217.557548:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573217.557552:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573217.557555:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573218.133497:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304821689, at 4304821696
00000100:00000010:1.0:1788573218.133515:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d80.
00000020:00000001:1.0:1788573219.605545:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573219.605549:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573219.605554:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573220.629634:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573220.629639:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573220.629643:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573222.293566:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573222.293578:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573222.293580:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573222.293585:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573222.293590:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573222.293598:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10153
00000400:00000200:1.0:1788573222.293607:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573222.293615:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573222.293618:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573222.293620:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293623:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573222.293625:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293627:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573222.293631:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573222.293632:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573222.293635:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573222.293638:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573222.293639:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573222.293641:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573222.293643:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293644:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573222.293646:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293647:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573222.293650:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573222.293652:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293654:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573222.293657:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573222.293661:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573222.293663:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573222.293665:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573222.293667:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573222.293678:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788573222.293682:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573222.293686:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573222.293687:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573222.293690:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573222.293693:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573222.293698:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573222.293706:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788573222.293714:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573222.293723:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393043072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573222.293732:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573222.293736:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573222.293739:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393043072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573222.293765:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573222.293769:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573222.293773:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573222.293774:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573222.293776:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573222.293780:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573222.293784:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10153
00000400:00000200:1.0:1788573222.293789:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573222.293793:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573222.293796:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573222.293802:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293806:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573222.293813:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293822:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573222.293830:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573222.293833:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573222.293843:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573222.293849:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573222.293856:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573222.293864:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573222.293889:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293899:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573222.293904:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293914:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573222.293925:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573222.293936:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573222.293947:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573222.293959:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573222.293967:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573222.293976:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573222.293986:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573222.293996:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573222.294030:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573222.294034:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573222.294044:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788573222.294048:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573222.294051:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573222.294056:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573222.294062:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573222.294063:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573222.294066:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573222.294073:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573222.294084:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573222.294101:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393043200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573222.294143:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573222.294152:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573222.294162:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393043200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1788573222.294202:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573222.294210:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1788573222.294233:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573222.294240:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573222.294252:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573222.294258:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573222.294261:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573222.294266:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573222.294275:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10153
00000400:00000200:1.0:1788573222.294283:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573222.294292:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573222.294301:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573222.294312:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.294321:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573222.294326:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.294333:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573222.294337:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573222.294342:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573222.294349:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573222.294355:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573222.294362:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573222.294369:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573222.294377:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.294382:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573222.294433:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573222.294437:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573222.294440:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573222.294443:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573222.294445:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573222.294458:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573222.294467:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573222.294471:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573222.294474:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573222.294481:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573222.294493:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573222.294496:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573222.294504:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788573222.294506:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573222.294509:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573222.294511:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573222.294515:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573222.294517:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573222.294520:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573222.294526:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788573222.294533:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573222.294542:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393043328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573222.294557:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573222.294561:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573222.294568:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393043328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573222.294585:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573222.294600:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573222.294617:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573222.294630:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10163)
00000800:00000200:0.1:1788573222.294829:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573222.294831:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573222.295042:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573222.295045:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788573222.677564:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573222.677568:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573222.677571:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573223.253523:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304826810, at 4304826816
00000100:00000010:1.0:1788573223.253542:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7840.
00000020:00000001:1.0:1788573223.701561:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573223.701565:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573223.701568:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573225.749609:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573225.749613:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573225.749615:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573226.773562:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573226.773565:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573226.773568:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573227.414054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573227.414058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573227.414435:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573227.414436:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573227.414674:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573227.414676:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:1.0:1788573228.373494:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304831929, at 4304831936
00000100:00000010:1.0:1788573228.373512:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7600.
00000020:00000001:1.0:1788573228.821581:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573228.821584:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573228.821587:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573229.845556:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573229.845561:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573229.845564:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573231.893634:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573231.893642:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573231.893646:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573232.534043:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573232.534049:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788573232.534706:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573232.534709:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788573232.535054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573232.535057:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:1.0:1788573232.917657:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573232.917661:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573232.917665:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573233.493520:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304837049, at 4304837056
00000100:00000010:1.0:1788573233.493536:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000020:00000001:1.0:1788573234.965596:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573234.965602:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573234.965606:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573235.989632:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573235.989635:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573235.989638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573237.653625:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573237.653641:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573237.653645:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573237.653649:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573237.653656:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573237.653664:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10168
00000400:00000200:1.0:1788573237.653676:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573237.653683:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573237.653694:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573237.653697:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653700:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573237.653702:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653704:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573237.653708:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573237.653710:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573237.653712:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573237.653715:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573237.653740:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573237.653742:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573237.653745:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653747:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573237.653748:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653750:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573237.653752:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573237.653755:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653757:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573237.653760:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573237.653763:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573237.653766:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573237.653768:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573237.653770:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573237.653778:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788573237.653781:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573237.653785:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573237.653786:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573237.653790:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573237.653793:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573237.653798:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573237.653805:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788573237.653812:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573237.653822:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393044224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573237.653831:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573237.653835:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573237.653839:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393044224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573237.653860:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573237.653862:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573237.653865:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573237.653867:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573237.653868:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573237.653871:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573237.653872:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10168
00000400:00000200:1.0:1788573237.653876:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573237.653878:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573237.653881:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573237.653883:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653885:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573237.653888:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653889:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573237.653891:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573237.653892:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573237.653893:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573237.653895:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573237.653897:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573237.653899:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573237.653904:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653906:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573237.653908:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653910:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573237.653911:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573237.653913:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573237.653915:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573237.653916:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573237.653922:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573237.653927:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573237.653933:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573237.653937:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573237.653945:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573237.653948:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573237.653952:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788573237.653954:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573237.653958:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573237.653960:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573237.653964:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573237.653966:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573237.653970:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573237.653979:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788573237.653988:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573237.654006:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393044352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573237.654025:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573237.654028:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573237.654038:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393044352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573237.654052:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573237.654057:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573237.654067:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573237.654069:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573237.654070:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573237.654072:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573237.654075:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10168
00000400:00000200:1.0:1788573237.654079:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573237.654086:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573237.654089:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573237.654094:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.654100:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573237.654103:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.654105:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573237.654108:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573237.654110:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573237.654114:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573237.654117:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573237.654121:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573237.654125:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573237.654137:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.654145:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573237.654151:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573237.654153:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573237.654158:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573237.654160:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573237.654163:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573237.654168:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573237.654177:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573237.654183:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573237.654186:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573237.654190:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573237.654203:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573237.654205:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573237.654208:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788573237.654213:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573237.654224:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573237.654227:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573237.654231:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573237.654235:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573237.654239:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573237.654245:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788573237.654256:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573237.654265:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393044480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573237.654276:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573237.654282:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573237.654289:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393044480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573237.654317:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573237.654322:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573237.654332:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573237.654345:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10178)
00000800:00000200:2.1:1788573237.654439:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573237.654442:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788573237.654770:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573237.654772:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788573237.655025:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573237.655028:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:1.0:1788573238.037645:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573238.037650:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573238.037653:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573238.613507:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304842170, at 4304842176
00000100:00000010:1.0:1788573238.613522:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000020:00000001:1.0:1788573239.061637:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573239.061643:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573239.061648:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573241.109620:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573241.109624:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573241.109628:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573242.133628:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573242.133631:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573242.133658:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573242.774462:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573242.774466:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788573242.774924:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573242.774928:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788573242.775245:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573242.775248:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000040:1.0:1788573243.733542:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304847290, at 4304847296
00000100:00000010:1.0:1788573243.733570:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000020:00000001:1.0:1788573244.181568:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573244.181572:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573244.181575:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573245.205570:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573245.205590:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573245.205594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573247.253554:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573247.253559:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573247.253562:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573247.893979:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573247.894209:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573247.894333:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788573248.277647:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573248.277654:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573248.277664:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573248.853490:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304852409, at 4304852416
00000100:00000010:1.0:1788573248.853505:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00000020:00000001:1.0:1788573250.325616:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573250.325620:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573250.325624:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573250.453473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573250.453502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573250.453503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573250.453508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573250.453511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573250.453521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573250.453523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573250.453525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573250.453529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573251.349556:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573251.349559:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573251.349562:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573251.477538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573251.477545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573251.477547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573251.477551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573251.477554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573251.477557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573251.477560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573251.477562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573251.477564:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573252.501492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573252.501500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573252.501503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573252.501508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573252.501512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573252.501516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573252.501518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573252.501522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573252.501525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573253.014124:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573253.014132:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573253.014733:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573253.014735:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573253.015071:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573253.015073:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788573253.397549:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573253.397553:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573253.397556:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573253.525521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573253.525527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573253.525529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573253.525533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573253.525537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573253.525539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573253.525542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573253.525544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573253.525546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573253.973517:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304857529, at 4304857536
00000100:00000010:1.0:1788573253.973533:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000800:00000200:1.0:1788573254.037643:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788573254.037649:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788573254.037653:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788573254.037773:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573254.037784:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573254.037790:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573254.037792:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788573254.421584:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573254.421589:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573254.421594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573254.549472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573254.549479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573254.549480:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573254.549485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573254.549487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573254.549491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573254.549493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573254.549495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573254.549497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573255.573504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573255.573510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573255.573512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573255.573516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573255.573519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573255.573522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573255.573525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573255.573527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573255.573529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573256.469611:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573256.469615:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573256.469619:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573256.597560:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573256.597568:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573256.597570:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573256.597575:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573256.597578:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573256.597583:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573256.597585:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573256.597588:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573256.597590:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573257.493629:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573257.493634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573257.493637:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573257.621530:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573257.621542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573257.621545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573257.621551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573257.621555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573257.621559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573257.621562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573257.621565:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573257.621570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573258.134376:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573258.134380:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573258.134915:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573258.134919:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573258.135256:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573258.135259:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788573258.645521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573258.645530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573258.645532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573258.645539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573258.645543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573258.645547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573258.645549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573258.645552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573258.645555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573259.093501:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304862650, at 4304862656
00000100:00000010:1.0:1788573259.093516:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f00.
00000020:00000001:1.0:1788573259.541580:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573259.541584:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573259.541587:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573259.669482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573259.669488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573259.669490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573259.669495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573259.669498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573259.669501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573259.669503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573259.669506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573259.669508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573260.565546:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573260.565553:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573260.565557:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573260.693544:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573260.693552:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573260.693556:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573260.693561:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573260.693565:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573260.693569:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573260.693572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573260.693575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573260.693577:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573261.717534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573261.717542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573261.717544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573261.717549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573261.717553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573261.717558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573261.717560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573261.717564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573261.717566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573262.613567:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573262.613572:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573262.613575:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573262.741456:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573262.741463:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573262.741465:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573262.741470:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573262.741473:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573262.741476:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573262.741478:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573262.741480:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573262.741482:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573263.254009:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573263.254017:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573263.254439:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573263.254441:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573263.254652:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573263.254655:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:1.0:1788573263.637548:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573263.637553:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573263.637557:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573263.765470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573263.765476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573263.765478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573263.765483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573263.765486:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573263.765489:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573263.765491:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573263.765495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573263.765497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573264.213493:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304867769, at 4304867776
00000100:00000010:1.0:1788573264.213510:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000100:00000001:1.0:1788573264.789498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573264.789505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573264.789507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573264.789512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573264.789515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573264.789519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573264.789521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573264.789523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573264.789525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573265.685528:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573265.685532:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573265.685535:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573265.813488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573265.813495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573265.813497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573265.813501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573265.813504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573265.813508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573265.813510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573265.813512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573265.813514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573266.709559:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573266.709563:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573266.709565:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573266.837458:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573266.837466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573266.837468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573266.837474:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573266.837478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573266.837482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573266.837485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573266.837488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573266.837491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573267.861563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573267.861570:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573267.861571:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573267.861577:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573267.861581:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573267.861584:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573267.861587:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573267.861590:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573267.861593:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573268.374046:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573268.374051:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788573268.374551:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573268.374554:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788573268.374886:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573268.374889:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:1.0:1788573268.757615:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573268.757620:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573268.757623:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573268.885533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573268.885541:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573268.885543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573268.885549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573268.885553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573268.885558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573268.885561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573268.885563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573268.885566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573269.333500:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304872889, at 4304872896
00000100:00000010:1.0:1788573269.333519:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000020:00000001:1.0:1788573269.781562:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573269.781567:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573269.781571:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573269.909540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573269.909558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573269.909561:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573269.909571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573269.909576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573269.909582:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573269.909586:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573269.909589:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573269.909592:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573270.933516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573270.933522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573270.933525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573270.933530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573270.933532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573270.933535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573270.933537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573270.933539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573270.933542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573271.829561:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573271.829565:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573271.829567:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573272.853549:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573272.853552:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573272.853555:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573273.494171:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573273.494176:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788573273.494796:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573273.494799:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788573273.495145:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573273.495148:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:1.0:1788573274.453499:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304878009, at 4304878016
00000100:00000010:1.0:1788573274.453517:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788573274.901561:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573274.901564:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573274.901567:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573275.925604:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573275.925608:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573275.925611:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573277.973546:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573277.973550:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573277.973553:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573278.614014:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573278.614019:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788573278.614507:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573278.614510:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788573278.614795:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573278.614798:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:1.0:1788573278.997574:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573278.997577:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573278.997579:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573279.573492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304883129, at 4304883136
00000100:00000010:1.0:1788573279.573499:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7a80.
00000020:00000001:1.0:1788573281.045636:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573281.045640:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573281.045644:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573282.069560:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573282.069563:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573282.069565:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573283.734175:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573283.734179:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1788573283.734633:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573283.734636:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573283.734903:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573283.734905:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:1.0:1788573284.117576:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573284.117581:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573284.117584:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573284.693532:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304888249, at 4304888256
00000100:00000010:1.0:1788573284.693547:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d78a0.
00000020:00000001:1.0:1788573285.141568:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573285.141571:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573285.141573:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573287.189534:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573287.189538:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573287.189541:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573288.213573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573288.213576:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573288.213579:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573288.854034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573288.854041:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573288.854434:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573288.854436:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573288.854652:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573288.854654:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:1.0:1788573289.813479:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304893369, at 4304893376
00000100:00000010:1.0:1788573289.813495:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000020:00000001:1.0:1788573290.261566:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573290.261569:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573290.261573:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573291.285538:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573291.285542:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573291.285544:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788573291.925481:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788573291.925491:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788573291.925497:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788573291.925649:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573291.925662:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573291.925667:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573291.925671:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788573293.333608:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573293.333613:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573293.333617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573293.974177:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573293.974181:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788573293.974559:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573293.974560:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788573293.974748:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573293.974750:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788573294.357659:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573294.357665:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573294.357669:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573294.933529:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304898489, at 4304898496
00000100:00000010:1.0:1788573294.933562:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000020:00000001:1.0:1788573296.405586:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573296.405589:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573296.405592:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573297.429586:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573297.429591:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573297.429595:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573299.094107:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573299.094113:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788573299.094601:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573299.094603:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788573299.094823:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573299.094825:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:1.0:1788573299.477561:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573299.477564:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573299.477567:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573300.053493:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304903609, at 4304903616
00000100:00000010:1.0:1788573300.053510:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00000020:00000001:1.0:1788573300.501567:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573300.501571:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573300.501574:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573302.549677:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573302.549683:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573302.549687:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573303.573620:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573303.573625:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573303.573629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573304.214006:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573304.214012:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573304.214387:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573304.214388:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573304.214651:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573304.214653:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:1.0:1788573305.173513:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304908729, at 4304908736
00000100:00000010:1.0:1788573305.173530:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7c00.
00000020:00000001:1.0:1788573305.621593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573305.621598:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573305.621601:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573306.645603:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573306.645607:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573306.645610:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573307.797508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573307.797516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573307.797518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573307.797525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573307.797528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573307.797532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573307.797536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573307.797538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573307.797542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573308.693617:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573308.693621:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573308.693625:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573308.821488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573308.821494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573308.821496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573308.821501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573308.821503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573308.821506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573308.821508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573308.821510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573308.821512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573309.334174:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573309.334178:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788573309.334631:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573309.334634:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788573309.334966:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573309.334969:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:1.0:1788573309.717619:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573309.717624:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573309.717629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573309.845517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573309.845525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573309.845526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573309.845531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573309.845535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573309.845539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573309.845542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573309.845544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573309.845547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573310.293536:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304913849, at 4304913856
00000100:00000010:1.0:1788573310.293555:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7cc0.
00000100:00000001:1.0:1788573310.869502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573310.869508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573310.869510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573310.869514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573310.869516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573310.869520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573310.869522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573310.869524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573310.869527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573311.765562:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573311.765567:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573311.765572:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573311.893504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573311.893510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573311.893512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573311.893516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573311.893519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573311.893523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573311.893525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573311.893527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573311.893529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573312.789616:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573312.789620:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573312.789624:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573312.917461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573312.917467:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573312.917470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573312.917475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573312.917484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573312.917487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573312.917489:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573312.917491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573312.917494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573313.941496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573313.941505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573313.941507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573313.941512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573313.941515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573313.941518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573313.941520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573313.941522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573313.941524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573314.454044:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573314.454242:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573314.454367:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788573314.837605:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573314.837610:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573314.837643:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573314.965475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573314.965484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573314.965487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573314.965493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573314.965497:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573314.965501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573314.965503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573314.965508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573314.965511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573315.413515:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304918969, at 4304918976
00000100:00000010:1.0:1788573315.413541:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000020:00000001:1.0:1788573315.861557:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573315.861560:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573315.861563:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573315.989475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573315.989480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573315.989482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573315.989486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573315.989489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573315.989492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573315.989495:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573315.989497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573315.989499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573317.013498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573317.013505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573317.013508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573317.013514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573317.013518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573317.013522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573317.013525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573317.013528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573317.013531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573317.909608:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573317.909611:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573317.909615:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573318.037495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573318.037501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573318.037503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573318.037507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573318.037510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573318.037513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573318.037515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573318.037517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573318.037520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573318.933602:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573318.933606:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573318.933609:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573319.061533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573319.061542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573319.061544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573319.061550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573319.061554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573319.061557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573319.061596:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573319.061599:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573319.061603:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573319.574505:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573319.574509:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573319.574971:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573319.574973:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573319.575251:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573319.575254:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:1.0:1788573320.085512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573320.085520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573320.085522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573320.085529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573320.085533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573320.085537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573320.085540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573320.085543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573320.085546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573320.533474:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304924090, at 4304924096
00000100:00000010:1.0:1788573320.533493:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000020:00000001:1.0:1788573320.981611:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573320.981614:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573320.981617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573321.109507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573321.109514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573321.109516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573321.109522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573321.109526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573321.109531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573321.109533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573321.109535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573321.109568:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573322.005643:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573322.005646:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573322.005649:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573322.133639:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573322.133649:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573322.133652:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573322.133657:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573322.133662:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573322.133666:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573322.133669:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573322.133672:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573322.133675:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573323.157492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573323.157501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573323.157503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573323.157510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573323.157545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573323.157548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573323.157551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573323.157554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573323.157558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573324.053609:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573324.053614:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573324.053617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573324.181533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573324.181540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573324.181543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573324.181548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573324.181552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573324.181556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573324.181558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573324.181562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573324.181565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573324.694025:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573324.694031:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573324.694348:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573324.694350:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573324.694589:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573324.694590:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:1.0:1788573325.077591:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573325.077601:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573325.077605:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573325.205521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573325.205532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573325.205534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573325.205539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573325.205541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573325.205544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573325.205546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573325.205548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573325.205550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573325.653563:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304929209, at 4304929216
00000100:00000010:1.0:1788573325.653581:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000100:00000001:1.0:1788573326.229563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573326.229574:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573326.229577:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573326.229585:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573326.229590:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573326.229595:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573326.229599:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573326.229602:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573326.229607:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573327.125641:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573327.125647:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573327.125659:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573327.253565:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573327.253576:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573327.253579:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573327.253585:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573327.253590:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573327.253596:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573327.253599:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573327.253603:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573327.253606:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573328.149643:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573328.149648:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573328.149652:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573328.277592:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573328.277598:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573328.277600:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573328.277604:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573328.277606:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573328.277609:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573328.277611:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573328.277613:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573328.277615:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573329.301545:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573329.301554:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573329.301557:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573329.301563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573329.301567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573329.301571:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573329.301574:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573329.301576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573329.301579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788573329.813552:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788573329.813560:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788573329.813573:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:1.0:1788573329.813756:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788573329.813762:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573329.813778:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573329.813786:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573329.813792:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788573329.814182:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573329.814187:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788573329.814694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573329.814697:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788573329.815007:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573329.815010:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:1.0:1788573330.197625:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573330.197630:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573330.197636:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573330.325500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573330.325513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573330.325516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573330.325537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573330.325542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573330.325547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573330.325550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573330.325554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573330.325559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573330.773491:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304934329, at 4304934336
00000100:00000010:1.0:1788573330.773511:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00000020:00000001:1.0:1788573331.221577:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573331.221581:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573331.221584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573331.349562:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573331.349571:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573331.349573:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573331.349578:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573331.349581:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573331.349586:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573331.349593:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573331.349596:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573331.349598:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573332.373476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573332.373484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573332.373487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573332.373494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573332.373498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573332.373502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573332.373505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573332.373507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573332.373510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573333.269617:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573333.269622:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573333.269625:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573333.397509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573333.397515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573333.397517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573333.397522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573333.397526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573333.397537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573333.397539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573333.397542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573333.397545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573334.293660:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573334.293664:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573334.293668:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573334.421498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573334.421504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573334.421506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573334.421510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573334.421513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573334.421517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573334.421519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573334.421522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573334.421526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573334.934017:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573334.934021:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788573334.934380:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573334.934381:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788573334.934658:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573334.934660:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:1.0:1788573335.445561:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573335.445573:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573335.445575:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573335.445581:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573335.445585:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573335.445590:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573335.445593:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573335.445596:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573335.445599:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573335.893513:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304939449, at 4304939456
00000100:00000010:1.0:1788573335.893530:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000020:00000001:1.0:1788573336.341664:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573336.341669:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573336.341672:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573336.469568:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573336.469578:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573336.469580:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573336.469587:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573336.469591:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573336.469594:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573336.469597:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573336.469601:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573336.469603:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573337.365695:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573337.365699:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573337.365703:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573337.493528:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573337.493535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573337.493537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573337.493543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573337.493548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573337.493551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573337.493554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573337.493557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573337.493560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573338.517587:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573338.517597:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573338.517599:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573338.517607:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573338.517612:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573338.517617:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573338.517620:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573338.517623:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573338.517627:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573339.413669:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573339.413673:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573339.413676:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573339.541503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573339.541512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573339.541514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573339.541519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573339.541522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573339.541526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573339.541529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573339.541531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573339.541534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573340.054209:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573340.054213:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788573340.054776:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573340.054779:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788573340.055133:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573340.055136:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:1.0:1788573340.437614:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573340.437619:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573340.437624:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573340.565535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573340.565544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573340.565546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573340.565553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573340.565556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573340.565561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573340.565564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573340.565567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573340.565569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573341.013525:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304944569, at 4304944576
00000100:00000010:1.0:1788573341.013546:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000020:00000001:1.0:1788573342.485634:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573342.485639:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573342.485642:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573343.509610:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573343.509614:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573343.509618:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573345.174085:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573345.174088:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573345.174424:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573345.174427:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573345.174677:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573345.174678:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:1.0:1788573345.557607:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573345.557612:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573345.557616:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573346.133504:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304949689, at 4304949696
00000100:00000010:1.0:1788573346.133525:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7de0.
00000020:00000001:1.0:1788573346.581579:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573346.581584:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573346.581587:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573348.629541:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573348.629544:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573348.629547:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573349.653677:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573349.653683:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573349.653688:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573350.294253:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573350.294264:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788573350.295020:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573350.295023:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788573350.295237:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573350.295239:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:1.0:1788573351.253483:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304954810, at 4304954816
00000100:00000010:1.0:1788573351.253495:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000020:00000001:1.0:1788573351.701587:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573351.701590:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573351.701594:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573352.725667:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573352.725673:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573352.725677:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573354.773617:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573354.773621:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573354.773624:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573355.414116:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573355.414119:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788573355.414552:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573355.414555:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788573355.414807:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573355.414809:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788573355.797541:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573355.797544:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573355.797547:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573356.373494:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304959929, at 4304959936
00000100:00000010:1.0:1788573356.373509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000020:00000001:1.0:1788573357.845596:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573357.845602:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573357.845606:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573358.869646:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573358.869649:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573358.869652:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573360.534254:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573360.534262:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788573360.534823:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573360.534826:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788573360.535080:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573360.535082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788573360.917598:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573360.917602:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573360.917605:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573361.493519:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304965050, at 4304965056
00000100:00000010:1.0:1788573361.493535:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000020:00000001:1.0:1788573361.941610:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573361.941614:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573361.941617:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573363.989634:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573363.989639:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573363.989642:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573365.013576:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573365.013579:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573365.013582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573365.654464:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573365.654470:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788573365.654977:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573365.654980:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788573365.655329:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573365.655332:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:1.0:1788573366.613516:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304970170, at 4304970176
00000100:00000010:1.0:1788573366.613536:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000020:00000001:1.0:1788573367.061629:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573367.061634:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573367.061638:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573368.085635:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573368.085638:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573368.085641:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788573369.749546:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788573369.749556:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788573369.749561:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788573369.749696:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573369.749710:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573369.749716:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573369.749720:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788573370.133635:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573370.133638:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573370.133641:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573370.774295:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573370.774304:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788573370.775068:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573370.775070:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788573370.775325:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573370.775328:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:1.0:1788573371.157594:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573371.157599:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573371.157603:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573371.733526:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304975290, at 4304975296
00000100:00000010:1.0:1788573371.733541:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000020:00000001:1.0:1788573373.205594:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573373.205598:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573373.205601:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573374.229586:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573374.229595:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573374.229598:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573375.894112:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573375.894116:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788573375.894591:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573375.894593:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788573375.894920:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573375.894923:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:1.0:1788573376.277636:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573376.277640:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573376.277644:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573376.853478:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304980409, at 4304980416
00000100:00000010:1.0:1788573376.853490:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d76c0.
00000020:00000001:1.0:1788573377.301609:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573377.301612:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573377.301615:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573379.349622:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573379.349625:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573379.349628:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573380.373608:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573380.373613:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573380.373616:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573381.014035:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573381.014291:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573381.014503:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788573381.973507:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304985529, at 4304985536
00000100:00000010:1.0:1788573381.973528:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ba0.
00000020:00000001:1.0:1788573382.421580:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573382.421585:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573382.421588:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573383.445585:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573383.445588:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573383.445591:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573385.493611:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573385.493615:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573385.493618:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573386.134698:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573386.134701:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573386.135083:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573386.135086:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573386.135459:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573386.135461:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788573386.517617:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573386.517621:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573386.517623:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573387.093493:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304990650, at 4304990656
00000100:00000010:1.0:1788573387.093509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7b40.
00000020:00000001:1.0:1788573388.565566:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573388.565571:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573388.565575:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573389.589647:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573389.589650:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573389.589653:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573391.254083:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573391.254089:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573391.254480:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573391.254481:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573391.254711:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573391.254713:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:1.0:1788573391.637594:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573391.637597:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573391.637600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573392.213495:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4304995769, at 4304995776
00000100:00000010:1.0:1788573392.213514:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000020:00000001:1.0:1788573392.661563:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573392.661566:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573392.661569:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573394.709546:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573394.709549:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573394.709551:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573395.733585:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573395.733588:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573395.733591:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573396.374098:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573396.374101:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573396.374612:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573396.374615:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573396.374936:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573396.374939:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:1.0:1788573397.333543:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305000889, at 4305000896
00000100:00000010:1.0:1788573397.333561:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000020:00000001:1.0:1788573397.781554:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573397.781557:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573397.781560:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573398.805604:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573398.805609:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573398.805613:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573400.853790:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573400.853793:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573400.853796:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573401.494087:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573401.494090:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788573401.494451:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573401.494453:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788573401.494696:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573401.494698:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:1.0:1788573401.877660:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573401.877665:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573401.877669:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573402.453506:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305006009, at 4305006016
00000100:00000010:1.0:1788573402.453522:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000020:00000001:1.0:1788573403.925656:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573403.925661:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573403.925666:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573404.949576:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573404.949580:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573404.949583:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573406.614046:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573406.614051:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788573406.614505:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573406.614508:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788573406.614737:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573406.614739:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:1.0:1788573406.997575:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573406.997578:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573406.997582:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573407.573513:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305011129, at 4305011136
00000100:00000010:1.0:1788573407.573534:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00000800:00000200:1.0:1788573407.637525:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788573407.637532:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788573407.637538:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788573407.637674:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573407.637688:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573407.637716:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573407.637720:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788573408.021589:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573408.021593:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573408.021596:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573410.069604:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573410.069607:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573410.069611:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573411.093600:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573411.093603:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573411.093607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573411.734038:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573411.734044:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788573411.734432:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573411.734435:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788573411.734670:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573411.734672:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000040:1.0:1788573412.693544:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305016249, at 4305016256
00000100:00000010:1.0:1788573412.693571:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000020:00000001:1.0:1788573413.141614:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573413.141620:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573413.141624:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573414.165579:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573414.165582:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573414.165585:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573416.213599:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573416.213604:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573416.213607:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573416.854026:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573416.854030:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788573416.854312:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573416.854314:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788573416.854604:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573416.854606:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:1.0:1788573417.237588:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573417.237592:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573417.237594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573417.813470:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305021369, at 4305021376
00000100:00000010:1.0:1788573417.813487:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000020:00000001:1.0:1788573419.285618:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573419.285621:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573419.285624:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573420.309589:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573420.309603:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573420.309610:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573421.974034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573421.974042:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788573421.974568:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573421.974569:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788573421.974816:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573421.974818:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788573422.357580:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573422.357583:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573422.357586:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573422.933501:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305026489, at 4305026496
00000100:00000010:1.0:1788573422.933516:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000020:00000001:1.0:1788573423.381572:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573423.381575:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573423.381578:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573425.429643:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573425.429648:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573425.429651:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573426.453590:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573426.453595:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573426.453600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573427.094180:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573427.094186:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788573427.094729:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573427.094731:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788573427.094928:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573427.094930:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:1.0:1788573428.053483:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305031609, at 4305031616
00000100:00000010:1.0:1788573428.053505:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000020:00000001:1.0:1788573428.501655:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573428.501658:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573428.501661:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573429.525566:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573429.525569:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573429.525572:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573431.573546:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573431.573550:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573431.573554:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573432.214041:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573432.214048:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788573432.214463:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573432.214464:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788573432.214733:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573432.214735:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:1.0:1788573432.597575:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573432.597580:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573432.597584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573433.173480:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305036729, at 4305036736
00000100:00000010:1.0:1788573433.173499:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000020:00000001:1.0:1788573434.645615:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573434.645618:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573434.645621:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573435.669573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573435.669578:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573435.669582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573437.334046:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573437.334051:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788573437.334559:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573437.334561:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788573437.334753:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573437.334755:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:1.0:1788573437.717648:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573437.717651:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573437.717654:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573438.293566:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305041849, at 4305041856
00000100:00000010:1.0:1788573438.293592:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000020:00000001:1.0:1788573438.741574:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573438.741580:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573438.741584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573440.789624:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573440.789628:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573440.789632:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573441.813630:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573441.813636:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573441.813642:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573442.453520:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573442.453529:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573442.453532:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573442.453536:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573442.453540:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573442.453547:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10373
00000400:00000200:1.0:1788573442.453556:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573442.453561:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573442.453564:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573442.453567:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453570:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573442.453573:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453575:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573442.453579:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573442.453581:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573442.453583:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573442.453586:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573442.453589:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573442.453592:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573442.453596:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453598:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573442.453599:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453601:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573442.453603:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573442.453605:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453607:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573442.453609:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573442.453612:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573442.453614:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573442.453615:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573442.453617:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573442.453626:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788573442.453628:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573442.453631:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573442.453632:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573442.453634:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573442.453636:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573442.453640:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573442.453646:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573442.453651:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573442.453658:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393059584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573442.453666:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573442.453669:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573442.453673:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393059584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573442.453694:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573442.453697:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573442.453699:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573442.453701:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573442.453702:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573442.453704:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573442.453706:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10373
00000400:00000200:1.0:1788573442.453708:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573442.453710:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573442.453712:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573442.453714:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453715:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573442.453717:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453719:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573442.453720:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573442.453721:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573442.453723:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573442.453724:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573442.453726:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573442.453727:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573442.453729:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453731:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573442.453732:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453734:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573442.453735:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573442.453737:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453739:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573442.453742:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573442.453748:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573442.453751:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573442.453754:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573442.453759:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573442.453762:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573442.453764:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573442.453771:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788573442.453774:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573442.453779:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573442.453781:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573442.453785:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573442.453788:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573442.453790:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573442.453794:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573442.453798:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573442.453805:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393059712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573442.453816:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573442.453820:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573442.453826:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393059712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573442.453840:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573442.453842:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573442.453848:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573442.453849:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573442.453850:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573442.453852:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573442.453854:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10373
00000400:00000200:1.0:1788573442.453856:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573442.453861:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573442.453863:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573442.453865:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453867:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573442.453871:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453874:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573442.453876:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573442.453880:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573442.453882:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573442.453883:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573442.453885:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573442.453890:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573442.453892:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453894:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573442.453899:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453903:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573442.453906:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573442.453910:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573442.453914:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573442.453918:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573442.453922:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573442.453926:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573442.453930:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573442.453933:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573442.453938:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573442.453943:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573442.453947:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788573442.453951:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573442.453954:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573442.453955:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573442.453959:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573442.453963:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573442.453965:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573442.453972:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573442.453979:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573442.453989:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393059840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573442.454010:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573442.454016:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573442.454021:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393059840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573442.454041:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573442.454045:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573442.454049:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573442.454055:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10383)
00000800:00000200:2.1:1788573442.454068:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573442.454076:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788573442.454468:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573442.454470:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788573442.454695:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573442.454697:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:1.0:1788573443.413462:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305046969, at 4305046976
00000100:00000010:1.0:1788573443.413470:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00000020:00000001:1.0:1788573443.861573:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573443.861576:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573443.861579:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573444.885575:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573444.885579:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573444.885582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788573446.549856:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573446.933614:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573446.933618:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573446.933621:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573447.573534:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573447.573545:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573447.573547:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573447.573552:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573447.573557:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573447.573564:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10378
00000400:00000200:1.0:1788573447.573572:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573447.573579:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573447.573582:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573447.573584:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573587:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573447.573589:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573591:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573447.573595:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573447.573596:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573447.573599:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573447.573602:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573447.573604:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573447.573605:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573447.573608:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573609:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573447.573611:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573613:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573447.573616:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573447.573619:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573621:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573447.573624:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573447.573627:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573447.573629:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573447.573631:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573447.573632:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573447.573640:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788573447.573643:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573447.573646:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573447.573647:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573447.573649:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573447.573652:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573447.573655:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573447.573661:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573447.573666:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573447.573674:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393059968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573447.573681:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573447.573685:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573447.573689:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393059968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573447.573708:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573447.573711:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573447.573713:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573447.573715:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573447.573716:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573447.573718:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573447.573720:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10378
00000400:00000200:1.0:1788573447.573722:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573447.573724:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573447.573726:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573447.573729:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573731:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573447.573733:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573735:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573447.573736:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573447.573737:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573447.573738:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573447.573739:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573447.573741:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573447.573743:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573447.573748:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573751:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573447.573754:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573758:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573447.573760:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573447.573764:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573767:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573447.573770:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573447.573776:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573447.573780:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573447.573783:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573447.573788:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573447.573792:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573447.573795:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573447.573799:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788573447.573804:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573447.573809:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573447.573812:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573447.573815:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573447.573818:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573447.573820:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573447.573824:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573447.573828:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573447.573837:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393060096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573447.573848:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573447.573852:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573447.573862:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393060096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573447.573886:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573447.573897:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573447.573904:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573447.573908:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573447.573911:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573447.573916:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573447.573922:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10378
00000400:00000200:1.0:1788573447.573930:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573447.573937:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573447.573940:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573447.573948:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573951:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573447.573958:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573961:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573447.573965:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573447.573967:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573447.573971:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573447.573973:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573447.573977:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573447.573982:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573447.573988:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573991:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573447.573996:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573447.573998:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573447.574000:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573447.574007:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573447.574013:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573447.574019:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573447.574024:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573447.574029:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573447.574034:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573447.574038:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573447.574045:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573447.574045:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788573447.574050:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573447.574056:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788573447.574059:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573447.574062:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573447.574063:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573447.574065:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573447.574067:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573447.574070:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573447.574079:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573447.574086:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573447.574095:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393060224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573447.574108:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573447.574112:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573447.574120:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393060224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573447.574128:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573447.574132:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573447.574138:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573447.574146:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10388)
00000800:00000200:2.1:1788573447.574282:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573447.574478:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788573447.957549:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573447.957553:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573447.957557:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573448.533504:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305052089, at 4305052096
00000100:00000010:1.0:1788573448.533521:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000020:00000001:1.0:1788573450.005600:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573450.005603:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573450.005606:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:1.0:1788573450.645701:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788573450.645742:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788573450.645750:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573450.645753:0:2352:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788573450.645757:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573450.645761:0:2352:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788573450.645764:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573450.645768:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:1.0:1788573450.645772:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573450.645775:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573450.645781:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65000
00000400:00000010:1.0:1788573450.645784:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65000.
00000100:00000001:1.0:1788573450.645788:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573450.645796:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f5f80 x1875444393059968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573450.645822:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573450.645827:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393059968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573472 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573450.645835:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573450.645838:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573450.645849:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573450.645851:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133700.
00000400:00000200:1.0:1788573450.645854:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573450.645858:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573450.645862:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:1.0:1788573450.645865:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:1.0:1788573450.645868:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573450.645875:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f4380 x1875444393060096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573450.645886:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573450.645893:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393060096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573472 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573450.645903:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573450.645908:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573450.645918:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573450.645923:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133e00.
00000400:00000200:1.0:1788573450.645928:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573450.645935:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573450.645940:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c654c8
00000400:00000010:1.0:1788573450.645945:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c654c8.
00000100:00000001:1.0:1788573450.645951:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573450.645960:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f4000 x1875444393060224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573450.645987:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573450.645994:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393060224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573472 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573450.646016:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573450.646021:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000001:1.0:1788573451.029594:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573451.029599:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573451.029603:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573452.694193:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573452.694200:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573452.694786:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573452.694788:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573452.695080:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573452.695082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788573453.077621:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573453.077625:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573453.077628:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573453.653510:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305057209, at 4305057216
00000100:00000010:1.0:1788573453.653529:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00000020:00000001:1.0:1788573454.101586:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573454.101591:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573454.101595:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573456.149626:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573456.149630:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573456.149634:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573457.173582:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573457.173585:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573457.173589:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573457.813513:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573457.813522:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573457.813524:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573457.813527:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573457.813531:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573457.813536:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10388
00000400:00000200:1.0:1788573457.813543:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573457.813548:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573457.813550:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573457.813554:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813556:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573457.813558:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813560:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573457.813562:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573457.813564:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573457.813566:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573457.813568:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573457.813570:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573457.813571:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573457.813573:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813575:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573457.813576:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813578:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573457.813580:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573457.813583:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813585:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573457.813587:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573457.813590:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573457.813592:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573457.813593:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573457.813595:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573457.813602:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788573457.813605:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573457.813607:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573457.813609:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573457.813623:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573457.813625:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573457.813629:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573457.813635:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573457.813641:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573457.813649:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393060736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573457.813660:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573457.813665:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573457.813670:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393060736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573457.813692:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573457.813696:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573457.813700:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573457.813701:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573457.813703:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573457.813706:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573457.813708:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10388
00000400:00000200:1.0:1788573457.813712:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573457.813714:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573457.813717:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573457.813719:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813721:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573457.813723:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813725:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573457.813727:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573457.813730:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573457.813732:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573457.813736:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573457.813739:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573457.813742:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573457.813752:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813757:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573457.813761:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813765:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573457.813769:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573457.813774:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813778:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573457.813783:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573457.813790:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573457.813797:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573457.813802:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573457.813809:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573457.813814:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573457.813819:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573457.813827:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788573457.813833:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573457.813838:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573457.813841:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573457.813844:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573457.813848:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573457.813852:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573457.813860:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788573457.813865:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573457.813874:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393060864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573457.813892:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573457.813894:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573457.813902:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393060864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573457.813913:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573457.813918:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573457.813923:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573457.813927:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573457.813929:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573457.813935:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573457.813942:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10388
00000400:00000200:1.0:1788573457.813949:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573457.813955:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573457.813960:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573457.813966:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813969:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573457.813974:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.813976:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573457.813979:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573457.813981:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573457.813984:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573457.813985:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573457.813987:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573457.813993:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573457.813998:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.814001:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573457.814005:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573457.814008:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573457.814012:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573457.814018:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573457.814020:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573457.814023:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573457.814027:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573457.814030:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573457.814032:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573457.814035:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573457.814039:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573457.814042:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573457.814047:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788573457.814051:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573457.814055:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573457.814057:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573457.814060:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573457.814061:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573457.814063:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573457.814070:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788573457.814074:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573457.814079:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:1.0:1788573457.814082:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393060992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1788573457.814084:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000001:1.0:1788573457.814095:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573457.814098:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573457.814104:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393060992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573457.814122:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573457.814128:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573457.814135:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573457.814145:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10399)
00000800:00000200:2.1:1788573457.814450:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573457.814453:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1788573457.814720:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573457.814722:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000040:1.0:1788573458.773506:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305062329, at 4305062336
00000100:00000010:1.0:1788573458.773525:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000020:00000001:1.0:1788573459.221585:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573459.221589:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573459.221592:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573460.245546:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573460.245550:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573460.245554:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573462.293570:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573462.293579:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573462.293582:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573462.933500:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573462.933512:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573462.933514:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573462.933518:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573462.933523:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573462.933530:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10394
00000400:00000200:1.0:1788573462.933540:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573462.933547:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573462.933550:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573462.933553:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933557:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573462.933561:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933563:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573462.933569:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573462.933571:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573462.933574:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573462.933578:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573462.933581:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573462.933584:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573462.933588:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933590:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573462.933592:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933595:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573462.933599:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573462.933629:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933631:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573462.933634:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573462.933637:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573462.933641:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573462.933644:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573462.933647:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573462.933661:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788573462.933666:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573462.933671:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573462.933673:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573462.933678:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573462.933682:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573462.933687:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573462.933698:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788573462.933705:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573462.933721:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393061120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573462.933730:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573462.933734:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573462.933738:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393061120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573462.933759:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573462.933761:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573462.933764:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573462.933766:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573462.933767:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573462.933770:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573462.933772:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10394
00000400:00000200:1.0:1788573462.933776:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573462.933778:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573462.933780:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573462.933784:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933786:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573462.933788:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933790:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573462.933792:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573462.933793:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573462.933794:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573462.933796:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573462.933797:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573462.933801:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573462.933808:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933811:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573462.933815:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933821:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573462.933823:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573462.933833:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573462.933837:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573462.933840:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573462.933848:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573462.933851:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573462.933852:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573462.933857:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573462.933862:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573462.933865:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573462.933875:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788573462.933887:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573462.933894:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573462.933896:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573462.933900:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573462.933906:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573462.933917:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573462.933928:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788573462.933939:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573462.933949:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393061248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573462.933964:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573462.933969:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573462.933973:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393061248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573462.933988:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573462.933991:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573462.934000:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573462.934001:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573462.934003:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573462.934005:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573462.934009:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10394
00000400:00000200:1.0:1788573462.934015:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573462.934034:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573462.934041:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573462.934046:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.934060:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573462.934069:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.934082:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573462.934089:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573462.934091:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573462.934094:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573462.934102:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573462.934106:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573462.934113:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573462.934119:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.934130:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573462.934135:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573462.934145:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573462.934149:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573462.934155:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573462.934165:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573462.934166:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573462.934173:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573462.934180:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573462.934182:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573462.934186:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573462.934192:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573462.934195:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573462.934200:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788573462.934205:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573462.934210:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573462.934212:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573462.934214:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573462.934217:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573462.934220:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573462.934227:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788573462.934230:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573462.934234:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393061376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573462.934240:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573462.934241:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573462.934247:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393061376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573462.934269:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573462.934274:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573462.934283:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573462.934290:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10404)
00000800:00000200:2.1:1788573462.934308:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573462.934311:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788573462.934748:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573462.934751:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788573462.935031:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573462.935033:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:1.0:1788573463.317649:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573463.317654:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573463.317658:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573463.893550:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305067450, at 4305067456
00000100:00000010:1.0:1788573463.893576:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000020:00000001:1.0:1788573465.365584:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573465.365587:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573465.365589:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573466.389625:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573466.389629:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573466.389633:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573468.053993:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573468.054001:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788573468.054372:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573468.054374:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788573468.054627:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573468.054630:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:1.0:1788573468.437605:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573468.437609:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573468.437613:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573469.013460:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305072569, at 4305072576
00000100:00000010:1.0:1788573469.013478:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000020:00000001:1.0:1788573469.461640:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573469.461644:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573469.461648:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573471.509642:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573471.509647:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573471.509652:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573472.533635:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573472.533640:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573472.533645:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573473.173503:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573473.173514:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573473.173517:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573473.173521:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573473.173525:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573473.173533:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10404
00000400:00000200:1.0:1788573473.173547:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573473.173555:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573473.173558:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573473.173562:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173566:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573473.173569:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173571:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573473.173576:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573473.173578:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573473.173594:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573473.173598:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573473.173600:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573473.173604:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573473.173608:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173610:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573473.173612:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173615:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573473.173618:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573473.173622:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173624:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573473.173627:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573473.173631:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573473.173635:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573473.173637:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573473.173640:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573473.173649:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788573473.173653:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573473.173656:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573473.173658:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573473.173662:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573473.173665:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573473.173668:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573473.173676:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788573473.173684:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573473.173694:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393061888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573473.173706:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573473.173710:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573473.173716:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393061888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573473.173741:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573473.173744:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573473.173747:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573473.173750:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573473.173752:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573473.173755:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573473.173758:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10404
00000400:00000200:1.0:1788573473.173763:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573473.173767:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573473.173770:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573473.173773:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173776:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573473.173779:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173781:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573473.173784:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573473.173788:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573473.173791:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573473.173796:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573473.173801:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573473.173805:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573473.173812:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173817:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573473.173822:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173827:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573473.173832:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573473.173839:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573473.173844:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573473.173848:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573473.173855:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573473.173859:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573473.173863:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573473.173869:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573473.173874:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573473.173879:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573473.173885:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788573473.173891:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573473.173895:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573473.173898:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573473.173902:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573473.173905:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573473.173909:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573473.173918:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573473.173924:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573473.173933:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393062016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573473.173945:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573473.173951:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573473.173964:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393062016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573473.173995:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573473.174000:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573473.174007:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573473.174011:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573473.174014:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573473.174018:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573473.174024:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10404
00000400:00000200:1.0:1788573473.174033:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573473.174038:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573473.174040:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573473.174045:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.174049:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573473.174051:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.174053:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573473.174057:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573473.174060:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573473.174062:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573473.174065:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573473.174067:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573473.174069:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573473.174075:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.174076:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573473.174080:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573473.174084:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573473.174086:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573473.174090:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573473.174096:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573473.174097:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573473.174101:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573473.174104:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573473.174106:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573473.174109:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573473.174111:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573473.174115:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573473.174118:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788573473.174120:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573473.174121:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573473.174122:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573473.174124:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573473.174125:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573473.174127:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573473.174132:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573473.174136:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573473.174142:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393062144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573473.174154:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573473.174158:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573473.174167:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393062144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573473.174180:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573473.174184:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573473.174191:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573473.174199:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10414)
00000800:00000200:2.1:1788573473.174209:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573473.174213:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788573473.174733:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573473.174736:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788573473.175056:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573473.175058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:1.0:1788573474.133474:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305077689, at 4305077696
00000100:00000010:1.0:1788573474.133487:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000020:00000001:1.0:1788573474.581535:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573474.581539:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573474.581541:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573475.605614:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573475.605618:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573475.605621:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573477.653651:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573477.653654:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573477.653657:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573478.293504:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573478.293515:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573478.293518:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573478.293522:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573478.293527:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573478.293534:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10409
00000400:00000200:1.0:1788573478.293543:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573478.293549:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573478.293557:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573478.293559:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293562:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573478.293564:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293566:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573478.293581:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573478.293584:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573478.293586:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573478.293589:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573478.293591:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573478.293593:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573478.293595:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293597:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573478.293598:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293599:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573478.293602:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573478.293604:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293606:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573478.293608:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573478.293612:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573478.293615:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573478.293617:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573478.293620:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573478.293630:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788573478.293634:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573478.293638:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573478.293639:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573478.293643:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573478.293645:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573478.293649:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573478.293658:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573478.293665:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573478.293673:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393062272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573478.293681:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573478.293684:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573478.293688:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393062272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573478.293709:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573478.293711:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573478.293714:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573478.293715:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573478.293717:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573478.293719:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573478.293720:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10409
00000400:00000200:1.0:1788573478.293724:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573478.293728:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573478.293730:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573478.293734:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293736:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573478.293739:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293742:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573478.293745:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573478.293746:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573478.293749:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573478.293754:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573478.293760:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573478.293764:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573478.293772:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293778:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573478.293782:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293785:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573478.293789:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573478.293793:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293797:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573478.293800:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573478.293807:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573478.293811:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573478.293814:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573478.293817:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573478.293823:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573478.293828:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573478.293834:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788573478.293839:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573478.293842:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573478.293844:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573478.293847:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573478.293849:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573478.293851:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573478.293859:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573478.293863:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573478.293871:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393062400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573478.293888:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573478.293893:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573478.293904:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393062400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573478.293924:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573478.293930:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573478.293935:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573478.293938:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573478.293939:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573478.293943:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573478.293948:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10409
00000400:00000200:1.0:1788573478.293956:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573478.293964:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573478.293969:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573478.293975:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293980:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573478.293985:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.293989:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573478.293995:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573478.293996:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573478.293998:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573478.294002:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573478.294008:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573478.294015:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573478.294023:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.294028:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573478.294035:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573478.294042:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.1:1788573478.294042:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:1.0:1788573478.294048:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:2.1:1788573478.294049:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00080000:1.0:1788573478.294057:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573478.294063:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573478.294068:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573478.294074:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573478.294081:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573478.294084:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573478.294089:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573478.294094:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573478.294099:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573478.294107:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788573478.294114:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573478.294120:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573478.294124:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573478.294129:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573478.294132:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573478.294135:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573478.294143:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573478.294149:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573478.294159:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393062528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573478.294178:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573478.294183:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573478.294193:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393062528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573478.294213:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573478.294221:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573478.294228:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573478.294240:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10419)
00000800:00000200:2.1:1788573478.294443:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573478.294446:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788573478.294715:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573478.294718:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:1.0:1788573478.677555:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573478.677558:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573478.677561:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573479.253520:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305082809, at 4305082816
00000100:00000010:1.0:1788573479.253546:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000020:00000001:1.0:1788573480.725559:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573480.725563:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573480.725577:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573481.749614:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573481.749618:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573481.749622:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573483.414116:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573483.414122:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788573483.414648:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573483.414652:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788573483.414993:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573483.414996:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:1.0:1788573483.797524:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573483.797528:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573483.797532:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573484.373492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305087929, at 4305087936
00000100:00000010:1.0:1788573484.373507:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000800:00000200:1.0:1788573484.437697:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573484.821612:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573484.821616:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573484.821620:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573486.869636:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573486.869642:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573486.869645:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573487.893612:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573487.893615:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573487.893618:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573488.534331:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573488.534336:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788573488.534716:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573488.534718:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788573488.534908:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573488.534909:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:1.0:1788573489.493512:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305093050, at 4305093056
00000100:00000010:1.0:1788573489.493532:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000020:00000001:1.0:1788573489.941575:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573489.941581:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573489.941584:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573490.965622:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573490.965626:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573490.965629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573493.013611:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573493.013616:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573493.013619:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573493.654077:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573493.654081:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788573493.654514:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573493.654515:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788573493.654780:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573493.654781:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788573494.037598:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573494.037603:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573494.037606:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573494.613507:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305098169, at 4305098176
00000100:00000010:1.0:1788573494.613525:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000020:00000001:1.0:1788573496.085591:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573496.085596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573496.085600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573497.109634:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573497.109638:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573497.109642:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573498.774161:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573498.774166:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788573498.774568:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573498.774570:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788573498.774802:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573498.774804:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000020:00000001:1.0:1788573499.157690:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573499.157697:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573499.157701:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573499.733563:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305103289, at 4305103296
00000100:00000010:1.0:1788573499.733584:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00000020:00000001:1.0:1788573500.181624:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573500.181631:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573500.181635:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573502.229794:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573502.229798:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573502.229800:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573503.253558:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573503.253562:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573503.253566:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573503.894110:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573503.894116:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788573503.894673:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573503.894675:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788573503.894913:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573503.894916:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00000040:1.0:1788573504.853514:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305108409, at 4305108416
00000100:00000010:1.0:1788573504.853542:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d20.
00000020:00000001:1.0:1788573505.301599:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573505.301603:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573505.301605:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573506.325542:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573506.325546:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573506.325549:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573508.373622:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573508.373627:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573508.373632:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573509.014287:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573509.014292:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788573509.014783:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573509.014785:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788573509.015094:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573509.015098:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:1.0:1788573509.397601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573509.397605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573509.397607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573509.973503:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305113529, at 4305113536
00000100:00000010:1.0:1788573509.973545:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7180.
00000020:00000001:1.0:1788573511.445621:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573511.445636:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573511.445640:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573512.469593:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573512.469598:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573512.469602:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573514.134034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573514.134226:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573514.134313:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788573514.517729:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573514.517735:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573514.517739:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573515.093528:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305118649, at 4305118656
00000100:00000010:1.0:1788573515.093537:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7240.
00000020:00000001:1.0:1788573515.541613:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573515.541618:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573515.541622:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573517.589616:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573517.589620:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573517.589622:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573518.613580:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573518.613583:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573518.613586:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573519.253923:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573519.253927:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573519.254440:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573519.254442:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573519.254762:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573519.254765:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:1.0:1788573520.213598:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305123769, at 4305123776
00000100:00000010:1.0:1788573520.213619:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f60.
00000020:00000001:1.0:1788573520.661605:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573520.661608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573520.661611:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573521.685667:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573521.685672:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573521.685675:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788573522.325761:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573523.733603:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573523.733606:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573523.733610:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573524.373535:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573524.373543:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573524.373545:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573524.373548:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573524.373551:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573524.373557:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10455
00000400:00000200:1.0:1788573524.373564:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573524.373569:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573524.373571:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573524.373573:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373575:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573524.373577:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373580:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573524.373583:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573524.373584:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573524.373586:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573524.373589:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573524.373590:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573524.373592:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573524.373594:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373595:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573524.373596:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373597:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573524.373600:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573524.373602:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373604:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573524.373605:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573524.373608:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573524.373610:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573524.373611:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573524.373614:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573524.373621:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788573524.373624:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573524.373626:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573524.373627:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573524.373629:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573524.373631:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573524.373634:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573524.373640:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573524.373645:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573524.373652:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393065728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573524.373659:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573524.373664:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573524.373668:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393065728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573524.373688:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573524.373692:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573524.373695:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573524.373698:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573524.373700:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573524.373704:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573524.373707:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10455
00000400:00000200:1.0:1788573524.373712:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573524.373718:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573524.373722:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573524.373729:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373734:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573524.373739:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373746:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573524.373753:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573524.373756:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573524.373761:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573524.373766:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573524.373770:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573524.373774:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573524.373781:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373786:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573524.373790:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373792:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573524.373796:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573524.373802:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373806:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573524.373809:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573524.373816:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573524.373820:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573524.373823:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573524.373826:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573524.373833:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573524.373839:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573524.373843:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788573524.373846:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573524.373850:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573524.373851:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573524.373853:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573524.373856:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573524.373859:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573524.373864:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573524.373867:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573524.373874:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393065856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573524.373885:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573524.373888:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573524.373894:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393065856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573524.373922:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573524.373926:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573524.373931:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573524.373933:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573524.373934:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573524.373936:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573524.373939:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10455
00000400:00000200:1.0:1788573524.373944:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573524.373947:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573524.373950:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573524.373953:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373954:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573524.373957:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373960:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573524.373965:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573524.373966:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573524.373967:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573524.373969:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573524.373971:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573524.373974:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573524.373981:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373985:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573524.373990:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573524.373996:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573524.374001:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573524.374008:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573524.374016:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573524.374021:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573524.374026:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573524.374033:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573524.374036:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573524.374038:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573524.374042:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573524.374046:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573524.374050:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788573524.374054:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573524.374058:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573524.374060:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573524.374063:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573524.374066:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573524.374069:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573524.374074:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573524.374078:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573524.374083:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:1.0:1788573524.374086:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393065984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1788573524.374087:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788573524.374102:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573524.374107:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573524.374112:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393065984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573524.374127:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573524.374130:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573524.374138:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573524.374143:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10465)
00000800:00000200:0.1:1788573524.374421:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573524.374423:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573524.374689:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573524.374691:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:1.0:1788573524.757569:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573524.757573:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573524.757576:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573525.333483:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305128889, at 4305128896
00000100:00000010:1.0:1788573525.333516:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7060.
00000020:00000001:1.0:1788573526.805575:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573526.805578:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573526.805580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573527.829607:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573527.829610:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573527.829612:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573529.494090:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573529.494094:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573529.494652:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573529.494655:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573529.494891:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573529.494893:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:1.0:1788573529.877563:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573529.877566:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573529.877569:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573530.453480:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305134009, at 4305134016
00000100:00000010:1.0:1788573530.453516:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7540.
00000020:00000001:1.0:1788573530.901664:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573530.901671:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573530.901677:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573532.949584:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573532.949590:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573532.949594:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573533.973631:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573533.973641:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573533.973645:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573534.613544:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573534.613559:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573534.613563:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573534.613568:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573534.613574:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573534.613584:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10465
00000400:00000200:1.0:1788573534.613596:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573534.613604:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573534.613608:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573534.613612:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613616:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573534.613619:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613623:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573534.613628:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573534.613630:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573534.613633:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573534.613645:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573534.613648:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573534.613651:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573534.613654:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613656:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573534.613659:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613661:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573534.613665:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573534.613670:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613673:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573534.613678:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573534.613683:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573534.613686:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573534.613688:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573534.613691:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573534.613702:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788573534.613705:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573534.613709:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573534.613711:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573534.613714:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573534.613717:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573534.613721:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573534.613730:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788573534.613737:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573534.613746:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393066496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573534.613755:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573534.613759:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573534.613762:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393066496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573534.613781:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573534.613784:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573534.613787:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573534.613788:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573534.613790:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573534.613792:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573534.613795:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10465
00000400:00000200:1.0:1788573534.613798:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573534.613800:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573534.613802:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573534.613804:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613806:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573534.613808:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613810:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573534.613811:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573534.613812:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573534.613814:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573534.613816:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573534.613817:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573534.613818:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573534.613820:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613821:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573534.613823:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613824:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573534.613826:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573534.613828:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613830:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573534.613832:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573534.613838:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573534.613840:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573534.613843:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573534.613848:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573534.613852:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573534.613856:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573534.613861:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788573534.613866:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573534.613871:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573534.613873:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573534.613877:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573534.613881:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573534.613885:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573534.613890:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788573534.613896:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573534.613907:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393066624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573534.613917:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573534.613920:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573534.613925:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393066624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573534.613933:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573534.613937:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573534.613941:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573534.613943:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573534.613946:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573534.613951:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573534.613955:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10465
00000400:00000200:1.0:1788573534.613958:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573534.613962:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573534.613967:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573534.613974:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613981:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573534.613988:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.613995:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573534.614000:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573534.614004:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573534.614008:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573534.614014:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573534.614017:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573534.614023:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573534.614027:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.614030:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573534.614033:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573534.614038:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573534.614040:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573534.614042:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573534.614044:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573534.614045:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573534.614048:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573534.614052:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573534.614055:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573534.614057:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573534.614059:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573534.614062:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573534.614069:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788573534.614073:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573534.614077:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573534.614080:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573534.614083:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573534.614084:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573534.614087:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573534.614092:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788573534.614098:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573534.614107:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393066752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573534.614113:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573534.614117:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573534.614125:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393066752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573534.614139:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573534.614142:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573534.614148:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573534.614153:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10475)
00000800:00000200:0.1:1788573534.614164:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573534.614168:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788573534.614539:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573534.614541:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788573534.614774:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573534.614776:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:1.0:1788573535.573531:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305139129, at 4305139136
00000100:00000010:1.0:1788573535.573550:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ea0.
00000020:00000001:1.0:1788573536.021663:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573536.021667:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573536.021671:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573537.045678:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573537.045683:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573537.045693:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573539.093665:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573539.093672:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573539.093677:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573539.734372:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573539.734381:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788573539.734814:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573539.734816:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788573539.735048:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573539.735050:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:1.0:1788573540.117598:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573540.117602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573540.117606:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573540.693548:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305144250, at 4305144256
00000100:00000010:1.0:1788573540.693573:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7120.
00000020:00000001:1.0:1788573542.165568:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573542.165572:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573542.165576:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573543.189614:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573543.189617:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573543.189619:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573544.854543:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573544.854547:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573544.855005:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573544.855007:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573544.855256:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573544.855258:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:1.0:1788573545.237608:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573545.237612:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573545.237617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573545.813535:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305149370, at 4305149376
00000100:00000010:1.0:1788573545.813557:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7120.
00000020:00000001:1.0:1788573546.261601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573546.261605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573546.261609:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573548.309591:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573548.309594:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573548.309597:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573549.333673:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573549.333677:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573549.333680:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573549.974187:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573549.974197:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573549.974703:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573549.974706:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573549.974921:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573549.974923:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:1.0:1788573550.933472:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305154489, at 4305154496
00000100:00000010:1.0:1788573550.933488:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ea0.
00000020:00000001:1.0:1788573551.381706:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573551.381714:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573551.381719:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573552.405651:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573552.405656:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573552.405667:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573554.453571:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573554.453576:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573554.453580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573555.094047:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573555.094053:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573555.094575:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573555.094577:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573555.094803:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573555.094805:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788573555.477602:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573555.477605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573555.477608:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573556.053530:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305159609, at 4305159616
00000100:00000010:1.0:1788573556.053564:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7540.
00000020:00000001:1.0:1788573557.525632:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573557.525637:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573557.525641:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573558.549698:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573558.549704:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573558.549708:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573560.213503:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573560.213515:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573560.213517:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573560.213526:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573560.213531:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573560.213549:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10491
00000400:00000200:1.0:1788573560.213558:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573560.213565:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573560.213568:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573560.213572:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213576:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573560.213579:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213582:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573560.213586:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573560.213589:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573560.213592:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573560.213595:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573560.213598:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573560.213600:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573560.213604:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213606:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573560.213608:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213611:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573560.213614:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573560.213618:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213620:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573560.213623:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573560.213627:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573560.213630:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573560.213632:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573560.213635:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573560.213645:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788573560.213649:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573560.213653:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573560.213654:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573560.213658:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573560.213661:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573560.213665:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573560.213673:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788573560.213681:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573560.213690:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393068416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573560.213701:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573560.213705:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573560.213711:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393068416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573560.213735:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573560.213738:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573560.213742:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573560.213744:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573560.213745:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573560.213748:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573560.213751:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10491
00000400:00000200:1.0:1788573560.213755:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573560.213759:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573560.213761:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573560.213765:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213768:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573560.213772:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213774:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573560.213777:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573560.213779:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573560.213781:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573560.213784:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573560.213788:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573560.213791:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573560.213796:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213800:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573560.213805:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213808:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573560.213814:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573560.213821:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573560.213829:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573560.213833:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573560.213841:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573560.213845:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573560.213850:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573560.213856:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573560.213861:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573560.213866:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573560.213874:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788573560.213878:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573560.213883:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573560.213887:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573560.213890:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573560.213894:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573560.213896:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573560.213902:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788573560.213908:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573560.213918:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393068544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573560.213931:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573560.213935:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573560.213943:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393068544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573560.213960:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573560.213967:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573560.213978:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573560.213982:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573560.213987:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573560.213994:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573560.213999:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10491
00000400:00000200:1.0:1788573560.214007:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573560.214013:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573560.214019:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573560.214026:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.214028:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573560.214034:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.214038:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573560.214043:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573560.214046:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573560.214049:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573560.214052:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573560.214057:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573560.214063:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573560.214070:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.214074:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573560.214079:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573560.214084:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573560.214089:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573560.214098:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573560.214104:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573560.214106:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573560.214110:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573560.214116:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573560.214120:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573560.214124:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573560.214130:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573560.214134:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573560.214140:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788573560.214147:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573560.214151:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573560.214153:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573560.214157:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573560.214160:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573560.214164:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573560.214171:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000800:00000200:0.1:1788573560.214176:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788573560.214178:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573560.214183:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00100000:1.0:1788573560.214189:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393068672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573560.214206:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573560.214208:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573560.214216:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393068672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573560.214231:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573560.214235:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573560.214242:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573560.214250:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10501)
00000800:00000200:0.1:1788573560.214716:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573560.214719:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573560.215091:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573560.215094:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788573560.597589:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573560.597593:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573560.597597:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573561.173509:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305164729, at 4305164736
00000100:00000010:1.0:1788573561.173526:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7060.
00000800:00000200:1.0:1788573561.237787:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573561.621653:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573561.621658:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573561.621662:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573563.669639:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573563.669642:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573563.669645:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573564.693644:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573564.693650:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573564.693654:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573565.334359:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573565.334363:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573565.334869:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573565.334872:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573565.335183:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573565.335186:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:1.0:1788573566.293459:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305169850, at 4305169856
00000100:00000010:1.0:1788573566.293470:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f60.
00000020:00000001:1.0:1788573566.741584:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573566.741587:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573566.741589:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573567.765531:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573567.765534:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573567.765537:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573569.813603:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573569.813608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573569.813612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573570.454325:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573570.454335:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573570.454953:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573570.454954:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573570.455177:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573570.455179:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:1.0:1788573570.837605:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573570.837610:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573570.837613:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573571.413492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305174970, at 4305174976
00000100:00000010:1.0:1788573571.413509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7240.
00000020:00000001:1.0:1788573572.885606:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573572.885611:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573572.885616:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573573.909632:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573573.909635:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573573.909638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573575.574333:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573575.574336:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788573575.574863:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573575.574866:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788573575.575201:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573575.575204:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:1.0:1788573575.957690:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573575.957696:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573575.957700:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573576.533530:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305180090, at 4305180096
00000100:00000010:1.0:1788573576.533552:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7180.
00000020:00000001:1.0:1788573576.981700:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573576.981707:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573576.981711:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573579.029613:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573579.029617:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573579.029621:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573580.053562:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573580.053587:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573580.053590:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573580.694087:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573580.694305:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573580.694533:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788573581.653495:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305185209, at 4305185216
00000100:00000010:1.0:1788573581.653515:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000020:00000001:1.0:1788573582.101648:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573582.101653:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573582.101676:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573583.125623:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573583.125627:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573583.125629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573585.173561:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573585.173566:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573585.173570:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573585.814135:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573585.814142:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573585.814720:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573585.814723:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573585.815037:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573585.815039:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788573586.197594:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573586.197597:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573586.197600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573586.773493:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305190329, at 4305190336
00000100:00000010:1.0:1788573586.773511:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d20.
00000020:00000001:1.0:1788573588.245659:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573588.245662:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573588.245665:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573589.269560:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573589.269563:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573589.269566:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573590.934250:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573590.934259:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573590.934786:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573590.934789:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573590.935087:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573590.935090:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:1.0:1788573591.317549:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573591.317552:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573591.317555:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573591.893516:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305195449, at 4305195456
00000100:00000010:1.0:1788573591.893532:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7b40.
00000020:00000001:1.0:1788573592.341586:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573592.341590:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573592.341593:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573594.389615:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573594.389618:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573594.389621:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573595.413633:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573595.413641:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573595.413646:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573596.054152:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573596.054160:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573596.054578:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573596.054580:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573596.054784:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573596.054786:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:1.0:1788573597.013498:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305200569, at 4305200576
00000100:00000010:1.0:1788573597.013510:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ba0.
00000020:00000001:1.0:1788573597.461569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573597.461572:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573597.461575:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573598.485582:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573598.485586:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573598.485590:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788573599.125692:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573600.533630:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573600.533636:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573600.533641:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573601.174279:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573601.174286:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788573601.174871:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573601.174873:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788573601.175186:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573601.175189:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:1.0:1788573601.557608:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573601.557615:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573601.557622:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573602.133507:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305205690, at 4305205696
00000100:00000010:1.0:1788573602.133528:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d76c0.
00000020:00000001:1.0:1788573603.605533:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573603.605537:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573603.605541:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573604.629641:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573604.629645:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573604.629649:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573606.294202:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573606.294211:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788573606.294804:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573606.294806:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788573606.295046:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573606.295048:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:1.0:1788573606.677589:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573606.677593:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573606.677597:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573607.253502:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305210810, at 4305210816
00000100:00000010:1.0:1788573607.253518:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7de0.
00000020:00000001:1.0:1788573607.701585:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573607.701589:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573607.701592:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573609.749622:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573609.749626:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573609.749630:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573610.773527:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573610.773530:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573610.773533:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573611.414096:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573611.414099:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788573611.414726:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573611.414729:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788573611.415054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573611.415057:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000040:1.0:1788573612.373461:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305215929, at 4305215936
00000100:00000010:1.0:1788573612.373486:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7cc0.
00000020:00000001:1.0:1788573612.821603:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573612.821609:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573612.821613:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573613.845580:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573613.845584:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573613.845587:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573615.893589:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573615.893593:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573615.893596:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573616.534045:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573616.534051:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788573616.534428:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573616.534430:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788573616.534646:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573616.534648:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:1.0:1788573616.917574:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573616.917577:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573616.917580:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573617.493494:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305221049, at 4305221056
00000100:00000010:1.0:1788573617.493511:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000020:00000001:1.0:1788573618.965549:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573618.965554:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573618.965558:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573619.989685:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573619.989689:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573619.989691:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573621.654218:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573621.654224:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1788573621.654637:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573621.654639:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1788573621.654872:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573621.654874:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000020:00000001:1.0:1788573622.037613:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573622.037616:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573622.037619:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573622.613479:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305226169, at 4305226176
00000100:00000010:1.0:1788573622.613496:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7c00.
00000020:00000001:1.0:1788573623.061515:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573623.061525:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573623.061530:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573625.109520:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573625.109524:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573625.109527:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573626.133568:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573626.133572:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573626.133576:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573626.774213:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573626.774217:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573626.774746:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573626.774748:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573626.774980:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573626.774982:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:1.0:1788573627.733485:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305231289, at 4305231296
00000100:00000010:1.0:1788573627.733504:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000020:00000001:1.0:1788573628.181615:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573628.181623:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573628.181629:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573629.205572:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573629.205577:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573629.205580:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573631.253573:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573631.253577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573631.253580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573631.894040:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573631.894044:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573631.894444:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573631.894445:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573631.894671:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573631.894673:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:1.0:1788573632.277555:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573632.277560:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573632.277564:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573632.853496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305236409, at 4305236416
00000100:00000010:1.0:1788573632.853509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d78a0.
00000020:00000001:1.0:1788573634.325590:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573634.325595:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573634.325599:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573635.349573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573635.349577:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573635.349579:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573635.477496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573635.477502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573635.477504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573635.477509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573635.477511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573635.477514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573635.477516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573635.477518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573635.477520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573636.501460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573636.501466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573636.501468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573636.501473:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573636.501476:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573636.501480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573636.501483:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573636.501486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573636.501490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573637.014077:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573637.014082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788573637.014543:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573637.014545:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788573637.014740:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573637.014742:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:1.0:1788573637.397607:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573637.397612:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573637.397615:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573637.525484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573637.525493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573637.525495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573637.525502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573637.525505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573637.525508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573637.525512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573637.525514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573637.525516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573637.973509:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305241529, at 4305241536
00000100:00000010:1.0:1788573637.973528:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000800:00000200:1.0:1788573638.037686:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573638.421528:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573638.421531:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573638.421533:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573638.549535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573638.549540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573638.549542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573638.549547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573638.549550:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573638.549553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573638.549555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573638.549557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573638.549560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573639.573528:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573639.573535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573639.573536:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573639.573541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573639.573544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573639.573547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573639.573550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573639.573552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573639.573554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573640.469587:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573640.469592:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573640.469595:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573640.597486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573640.597494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573640.597497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573640.597502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573640.597506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573640.597509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573640.597512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573640.597515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573640.597518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573641.493555:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573641.493559:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573641.493563:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573641.621467:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573641.621474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573641.621476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573641.621481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573641.621484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573641.621487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573641.621489:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573641.621492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573641.621494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573642.134118:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573642.134126:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788573642.134679:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573642.134682:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788573642.134992:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573642.134995:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:1.0:1788573642.645536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573642.645545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573642.645547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573642.645553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573642.645558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573642.645561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573642.645564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573642.645567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573642.645570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573643.093517:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305246649, at 4305246656
00000100:00000010:1.0:1788573643.093539:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000020:00000001:1.0:1788573643.541573:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573643.541577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573643.541580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573643.669463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573643.669469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573643.669471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573643.669475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573643.669478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573643.669480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573643.669482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573643.669484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573643.669486:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573644.565569:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573644.565573:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573644.565576:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573646.613658:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573646.613664:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573646.613667:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573647.253543:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573647.253554:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573647.253557:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573647.253561:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573647.253566:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573647.253573:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10578
00000400:00000200:1.0:1788573647.253583:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573647.253590:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573647.253594:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573647.253597:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253599:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573647.253601:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253603:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573647.253608:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573647.253610:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573647.253612:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573647.253616:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573647.253618:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573647.253620:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573647.253621:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253623:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573647.253624:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253625:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573647.253627:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573647.253630:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253632:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573647.253634:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573647.253637:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573647.253640:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573647.253641:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573647.253644:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573647.253654:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788573647.253657:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573647.253661:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573647.253662:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573647.253665:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573647.253668:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573647.253673:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573647.253681:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788573647.253688:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573647.253697:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393074944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573647.253721:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573647.253725:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573647.253729:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393074944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573647.253750:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573647.253753:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573647.253756:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573647.253757:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573647.253758:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573647.253760:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573647.253762:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10578
00000400:00000200:1.0:1788573647.253765:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573647.253767:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573647.253769:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573647.253772:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253774:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573647.253777:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253779:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573647.253781:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573647.253782:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573647.253784:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573647.253787:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573647.253789:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573647.253791:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573647.253795:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253798:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573647.253800:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253803:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573647.253806:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573647.253810:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573647.253814:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573647.253819:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573647.253827:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573647.253833:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573647.253838:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573647.253845:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573647.253852:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573647.253857:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573647.253861:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788573647.253865:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573647.253869:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573647.253871:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573647.253874:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573647.253878:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573647.253881:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573647.253887:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573647.253893:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573647.253900:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393075072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573647.253913:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573647.253915:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573647.253920:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393075072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573647.253930:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573647.253933:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573647.253937:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573647.253941:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573647.253942:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573647.253978:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573647.253988:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10578
00000400:00000200:1.0:1788573647.254000:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573647.254010:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573647.254021:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573647.254030:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.254036:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573647.254043:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.254046:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573647.254048:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573647.254049:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573647.254053:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573647.254057:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573647.254061:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573647.254063:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573647.254069:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.254078:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573647.254082:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573647.254087:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573647.254092:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573647.254099:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573647.254106:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573647.254111:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573647.254117:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573647.254124:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573647.254131:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573647.254137:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573647.254143:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573647.254145:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573647.254150:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788573647.254157:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573647.254167:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573647.254171:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573647.254175:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573647.254178:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573647.254181:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573647.254188:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573647.254198:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573647.254207:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:1.0:1788573647.254227:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393075200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573647.254238:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573647.254240:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573647.254245:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393075200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573647.254267:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573647.254273:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573647.254297:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573647.254308:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10588)
00000800:00000200:2.1:1788573647.254524:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573647.254629:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788573647.637676:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573647.637684:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573647.637689:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573648.213536:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305251769, at 4305251776
00000100:00000010:1.0:1788573648.213568:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00000020:00000001:1.0:1788573649.685609:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573649.685616:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573649.685619:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573650.709674:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573650.709678:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573650.709682:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573652.374170:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573652.374179:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573652.374795:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573652.374797:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573652.375105:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573652.375107:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788573652.757553:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573652.757558:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573652.757562:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573652.885517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573652.885523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573652.885525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573652.885530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573652.885534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573652.885537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573652.885540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573652.885544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573652.885547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573653.333488:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305256889, at 4305256896
00000100:00000010:1.0:1788573653.333507:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000020:00000001:1.0:1788573653.781544:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573653.781548:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573653.781551:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573653.909518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573653.909537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573653.909539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573653.909543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573653.909546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573653.909549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573653.909551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573653.909553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573653.909559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573654.933480:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573654.933486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573654.933488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573654.933493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573654.933496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573654.933499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573654.933501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573654.933503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573654.933505:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573655.829568:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573655.829571:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573655.829574:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573655.957441:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573655.957449:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573655.957452:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573655.957457:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573655.957460:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573655.957464:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573655.957467:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573655.957470:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573655.957473:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573656.853627:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573656.853631:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573656.853643:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573656.981471:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573656.981478:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573656.981479:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573656.981484:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573656.981486:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573656.981490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573656.981492:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573656.981493:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573656.981518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573657.494115:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573657.494121:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573657.494711:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573657.494714:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573657.495028:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573657.495031:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788573658.005458:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573658.005465:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573658.005467:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573658.005485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573658.005488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573658.005491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573658.005493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573658.005495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573658.005497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573658.453512:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305262009, at 4305262016
00000100:00000010:1.0:1788573658.453528:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00000020:00000001:1.0:1788573658.901607:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573658.901610:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573658.901613:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573659.029481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573659.029490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573659.029493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573659.029498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573659.029502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573659.029506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573659.029509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573659.029513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573659.029516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573659.925575:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573659.925579:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573659.925582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573660.053515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573660.053522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573660.053523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573660.053529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573660.053533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573660.053537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573660.053540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573660.053543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573660.053546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573661.077513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573661.077521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573661.077523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573661.077528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573661.077531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573661.077537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573661.077539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573661.077542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573661.077545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573661.973549:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573661.973554:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573661.973557:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573662.101499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573662.101507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573662.101510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573662.101515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573662.101519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573662.101523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573662.101525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573662.101528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573662.101532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573662.614021:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573662.614027:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573662.614548:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573662.614549:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573662.614734:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573662.614735:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:1.0:1788573662.997567:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573662.997571:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573662.997574:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573663.125443:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573663.125449:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573663.125451:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573663.125455:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573663.125457:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573663.125461:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573663.125463:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573663.125465:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573663.125467:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573663.573466:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305267129, at 4305267136
00000100:00000010:1.0:1788573663.573481:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000100:00000001:1.0:1788573664.149507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573664.149515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573664.149518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573664.149524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573664.149528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573664.149532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573664.149535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573664.149537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573664.149540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573665.045560:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573665.045565:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573665.045569:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573665.173462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573665.173469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573665.173471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573665.173475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573665.173478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573665.173481:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573665.173483:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573665.173485:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573665.173488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573666.069631:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573666.069637:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573666.069642:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573666.197518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573666.197537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573666.197539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573666.197546:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573666.197550:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573666.197553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573666.197557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573666.197560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573666.197563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573667.221547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573667.221555:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573667.221557:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573667.221562:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573667.221566:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573667.221569:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573667.221572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573667.221575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573667.221578:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573667.733509:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573667.733524:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573667.733528:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573667.733533:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573667.733539:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573667.733549:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10598
00000400:00000200:1.0:1788573667.733560:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573667.733568:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573667.733587:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573667.733591:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733594:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573667.733597:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733600:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573667.733604:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573667.733607:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573667.733611:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573667.733614:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573667.733617:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573667.733620:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573667.733623:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733626:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573667.733629:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733631:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573667.733635:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573667.733639:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733643:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573667.733647:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573667.733652:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573667.733656:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573667.733659:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573667.733662:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573667.733674:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788573667.733677:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573667.733682:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573667.733683:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573667.733687:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573667.733690:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573667.733695:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573667.733703:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573667.733710:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573667.733721:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393076480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573667.733749:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573667.733755:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573667.733761:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393076480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573667.733788:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573667.733792:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573667.733796:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573667.733798:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573667.733799:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573667.733803:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573667.733809:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10598
00000400:00000200:1.0:1788573667.733814:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573667.733818:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573667.733824:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573667.733831:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733836:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573667.733842:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733845:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573667.733849:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573667.733850:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573667.733852:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573667.733854:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573667.733857:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573667.733859:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573667.733866:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733870:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573667.733873:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733879:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573667.733886:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573667.733894:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573667.733902:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573667.733909:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573667.733920:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573667.733927:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573667.733931:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573667.733938:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573667.733946:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573667.733952:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573667.733964:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788573667.733973:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573667.733992:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573667.733997:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573667.734007:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573667.734010:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573667.734015:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573667.734022:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573667.734029:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573667.734043:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393076608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573667.734062:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573667.734070:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573667.734080:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393076608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573667.734140:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573667.734145:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573667.734155:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573667.734163:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573667.734167:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573667.734171:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573667.734176:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10598
00000400:00000200:1.0:1788573667.734187:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573667.734199:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573667.734207:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573667.734222:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.734251:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573667.734256:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.734266:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573667.734271:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573667.734274:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573667.734279:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573667.734286:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573667.734291:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573667.734299:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573667.734313:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.734323:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573667.734330:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573667.734338:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573667.734341:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573667.734347:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573667.734356:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573667.734366:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573667.734383:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:2.1:1788573667.734434:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1788573667.734435:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573667.734444:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:2.1:1788573667.734444:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000008:00100000:1.0:1788573667.734451:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573667.734459:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573667.734471:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573667.734482:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788573667.734488:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573667.734494:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573667.734499:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573667.734505:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573667.734516:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573667.734521:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573667.734528:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573667.734533:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573667.734547:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393076736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573667.734567:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573667.734574:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573667.734593:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393076736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573667.734618:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573667.734625:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573667.734632:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573667.734641:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 3 (10608)
00000800:00000200:2.1:1788573667.734881:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573667.734883:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788573667.735073:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573667.735075:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:1.0:1788573668.117618:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573668.117622:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573668.117626:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573668.245602:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573668.245611:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573668.245613:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573668.245618:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573668.245622:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573668.245626:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573668.245629:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573668.245632:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573668.245635:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573668.693518:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305272250, at 4305272256
00000100:00000010:1.0:1788573668.693540:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000020:00000001:1.0:1788573669.141629:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573669.141634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573669.141638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573669.269546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573669.269555:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573669.269559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573669.269566:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573669.269570:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573669.269577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573669.269580:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573669.269584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573669.269587:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573670.293449:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573670.293455:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573670.293457:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573670.293461:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573670.293464:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573670.293467:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573670.293469:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573670.293471:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573670.293474:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573670.741960:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573670.741966:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 39 secs later)
00000800:00000200:2.1:1788573670.742333:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573670.742336:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 39 secs later)
00000800:00000200:2.1:1788573670.742570:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573670.742572:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 39 secs later)
00000020:00000001:1.0:1788573671.189536:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573671.189540:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573671.189544:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573671.317498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573671.317506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573671.317508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573671.317514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573671.317518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573671.317522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573671.317525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573671.317528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573671.317531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573671.701495:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305275257, at 4305275264
00000100:00000010:1.0:1788573671.701513:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000020:00000001:1.0:1788573672.213583:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573672.213586:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573672.213589:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573672.341458:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573672.341462:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573672.341464:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573672.341468:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573672.341471:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573672.341474:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573672.341477:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573672.341479:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573672.341481:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573674.261588:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573674.261593:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573674.261596:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573675.285541:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573675.285544:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573675.285547:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788573675.925736:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788573675.925751:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573675.925764:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573675.925773:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573675.925779:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788573675.926069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573675.926077:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 34 secs later)
00000800:00000200:2.1:1788573675.926364:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573675.926366:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 34 secs later)
00000800:00000200:2.1:1788573675.926642:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573675.926644:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 34 secs later)
00000100:00000040:1.0:1788573676.885460:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305280441, at 4305280448
00000100:00000010:1.0:1788573676.885474:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000020:00000001:1.0:1788573677.333602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573677.333605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573677.333608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573678.357560:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573678.357564:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573678.357568:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573680.405582:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573680.405585:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573680.405588:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573681.046115:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573681.046118:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 29 secs later)
00000800:00000200:2.1:1788573681.046477:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573681.046479:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 29 secs later)
00000800:00000200:2.1:1788573681.046673:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573681.046675:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 29 secs later)
00000020:00000001:1.0:1788573681.429554:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573681.429558:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573681.429561:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573682.005490:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305285561, at 4305285568
00000100:00000010:1.0:1788573682.005515:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000020:00000001:1.0:1788573683.477569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573683.477573:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573683.477576:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573684.501656:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573684.501662:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573684.501665:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573686.166017:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573686.166023:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 24 secs later)
00000800:00000200:2.1:1788573686.166510:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573686.166512:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 24 secs later)
00000800:00000200:2.1:1788573686.166746:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573686.166748:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 24 secs later)
00000020:00000001:1.0:1788573686.549568:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573686.549573:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573686.549576:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573687.125481:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305290681, at 4305290688
00000100:00000010:1.0:1788573687.125498:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000020:00000001:1.0:1788573687.573608:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573687.573611:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573687.573614:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573689.621556:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573689.621559:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573689.621562:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573690.645681:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573690.645685:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573690.645688:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573691.286168:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573691.286177:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 19 secs later)
00000800:00000200:2.1:1788573691.286702:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573691.286704:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 19 secs later)
00000800:00000200:2.1:1788573691.286889:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573691.286891:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 19 secs later)
00000100:00000040:1.0:1788573692.245538:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305295801, at 4305295808
00000100:00000010:1.0:1788573692.245555:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000020:00000001:1.0:1788573692.693643:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573692.693649:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573692.693653:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573693.717572:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573693.717598:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573693.717601:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573695.765713:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573695.765719:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573695.765722:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573696.406030:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573696.406036:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 14 secs later)
00000800:00000200:2.1:1788573696.406473:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573696.406475:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 14 secs later)
00000800:00000200:2.1:1788573696.406758:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573696.406762:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 14 secs later)
00000020:00000001:1.0:1788573696.789623:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573696.789627:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573696.789631:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573697.365512:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305300921, at 4305300928
00000100:00000010:1.0:1788573697.365526:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000020:00000001:1.0:1788573698.837580:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573698.837583:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573698.837587:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573699.861619:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573699.861622:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573699.861625:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573701.526033:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573701.526039:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 9 secs later)
00000800:00000200:0.1:1788573701.526463:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573701.526464:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 9 secs later)
00000800:00000200:0.1:1788573701.526657:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573701.526659:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 9 secs later)
00000020:00000001:1.0:1788573701.909588:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573701.909593:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573701.909597:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573702.485521:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305306041, at 4305306048
00000100:00000010:1.0:1788573702.485536:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7a80.
00000020:00000001:1.0:1788573702.933671:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573702.933676:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573702.933680:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573704.981554:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573704.981558:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573704.981562:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573706.005559:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573706.005563:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573706.005566:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573706.646111:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573706.646114:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 4 secs later)
00000800:00000200:0.1:1788573706.646528:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573706.646530:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 4 secs later)
00000800:00000200:0.1:1788573706.646782:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573706.646784:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 4 secs later)
00000100:00000040:1.0:1788573707.605486:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305311161, at 4305311168
00000100:00000010:1.0:1788573707.605511:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788573708.053590:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573708.053596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573708.053600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573709.077625:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573709.077631:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573709.077635:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573711.125580:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573711.125584:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573711.125588:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573711.766142:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573711.766380:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573711.766565:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788573712.149637:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573712.149643:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573712.149646:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573712.725500:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305316281, at 4305316288
00000100:00000010:1.0:1788573712.725517:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000800:00000200:1.0:1788573713.813703:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573714.197609:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573714.197614:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573714.197618:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:1.0:1788573714.837663:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788573714.837713:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788573714.837722:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573714.837727:0:2352:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788573714.837732:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573714.837736:0:2352:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788573714.837741:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573714.837745:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:1.0:1788573714.837751:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573714.837757:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573714.837763:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65110
00000400:00000010:1.0:1788573714.837784:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65110.
00000100:00000001:1.0:1788573714.837790:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573714.837800:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348710a80 x1875444393079936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573714.837826:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573714.837831:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393079936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573714.837840:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573714.837842:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573714.837852:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573714.837855:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133c00.
00000400:00000200:1.0:1788573714.837858:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573714.837863:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573714.837866:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:1.0:1788573714.837869:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:1.0:1788573714.837873:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573714.837878:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348713800 x1875444393080064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573714.837888:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573714.837895:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393080064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573714.837907:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573714.837910:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573714.837919:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573714.837925:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:1.0:1788573714.837931:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573714.837937:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573714.837940:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c654c8
00000400:00000010:1.0:1788573714.837945:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c654c8.
00000100:00000001:1.0:1788573714.837949:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573714.837959:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348710700 x1875444393080192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573714.837972:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573714.837979:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393080192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573714.837990:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573714.837993:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000001:1.0:1788573715.221568:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573715.221573:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573715.221577:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573716.886263:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573716.886271:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573716.886834:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573716.886837:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573716.887174:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573716.887177:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788573717.269608:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573717.269613:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573717.269616:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573717.845493:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305321402, at 4305321408
00000100:00000010:1.0:1788573717.845509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000020:00000001:1.0:1788573718.293549:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573718.293554:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573718.293559:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573720.341542:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573720.341547:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573720.341551:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573721.365539:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573721.365541:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573721.365545:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573722.005515:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573722.005523:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573722.005526:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573722.005529:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573722.005532:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573722.005538:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10653
00000400:00000200:1.0:1788573722.005545:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573722.005550:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573722.005553:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573722.005555:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005558:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573722.005559:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005561:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573722.005564:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573722.005566:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573722.005568:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573722.005571:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573722.005574:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573722.005577:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573722.005580:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005582:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573722.005584:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005586:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573722.005589:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573722.005592:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005595:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573722.005598:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573722.005602:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573722.005605:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573722.005607:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573722.005610:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573722.005621:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788573722.005625:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573722.005629:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573722.005630:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573722.005633:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573722.005636:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573722.005639:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573722.005646:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788573722.005652:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573722.005660:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393080704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573722.005668:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573722.005671:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573722.005674:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393080704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573722.005691:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573722.005694:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573722.005697:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573722.005698:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573722.005699:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573722.005701:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573722.005704:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10653
00000400:00000200:1.0:1788573722.005708:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573722.005710:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573722.005712:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573722.005715:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005720:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573722.005723:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005728:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573722.005732:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573722.005733:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573722.005734:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573722.005736:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573722.005738:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573722.005739:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573722.005743:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005749:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573722.005753:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005759:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573722.005763:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573722.005768:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005773:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573722.005777:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573722.005785:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573722.005788:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573722.005792:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573722.005797:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573722.005800:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573722.005802:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573722.005809:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788573722.005813:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573722.005818:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573722.005820:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573722.005825:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573722.005826:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573722.005827:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573722.005832:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573722.005836:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573722.005845:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393080832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573722.005864:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573722.005868:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573722.005877:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393080832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573722.005889:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573722.005892:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573722.005894:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573722.005895:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573722.005897:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573722.005924:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573722.005928:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10653
00000400:00000200:1.0:1788573722.005936:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573722.005942:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573722.005946:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573722.005951:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005955:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573722.005960:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005963:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573722.005967:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573722.005969:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573722.005972:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573722.005975:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573722.005979:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573722.005982:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573722.005987:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005989:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573722.005990:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005991:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573722.005993:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573722.005996:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573722.005998:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573722.005999:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573722.006001:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573722.006005:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573722.006007:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573722.006011:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573722.006015:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573722.006019:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573722.006025:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788573722.006028:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573722.006030:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573722.006033:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573722.006035:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573722.006036:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573722.006039:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573722.006046:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573722.006051:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573722.006052:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573722.006055:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00100000:1.0:1788573722.006058:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393080960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573722.006071:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573722.006074:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573722.006080:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393080960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573722.006091:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573722.006095:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573722.006100:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573722.006106:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10663)
00000800:00000200:2.1:1788573722.006360:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573722.006361:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573722.006592:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573722.006594:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:1.0:1788573722.965485:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305326521, at 4305326528
00000100:00000010:1.0:1788573722.965500:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000020:00000001:1.0:1788573723.413538:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573723.413541:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573723.413545:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573724.437580:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573724.437583:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573724.437585:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573726.485589:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573726.485592:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573726.485594:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573727.125496:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573727.125507:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573727.125509:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573727.125512:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573727.125515:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573727.125525:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10658
00000400:00000200:1.0:1788573727.125532:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573727.125538:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573727.125540:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573727.125542:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125545:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573727.125547:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125549:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573727.125552:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573727.125554:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573727.125556:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573727.125559:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573727.125563:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573727.125565:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573727.125567:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125569:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573727.125570:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125572:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573727.125574:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573727.125576:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125578:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573727.125581:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573727.125584:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573727.125587:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573727.125589:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573727.125591:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573727.125600:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788573727.125603:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573727.125608:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573727.125610:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573727.125612:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573727.125615:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573727.125618:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573727.125625:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573727.125630:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573727.125637:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393081088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573727.125645:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573727.125659:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573727.125662:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393081088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573727.125678:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573727.125681:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573727.125683:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573727.125684:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573727.125686:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573727.125688:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573727.125690:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10658
00000400:00000200:1.0:1788573727.125692:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573727.125695:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573727.125697:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573727.125700:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125701:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573727.125704:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125709:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573727.125711:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573727.125714:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573727.125716:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573727.125719:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573727.125722:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573727.125725:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573727.125733:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125737:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573727.125741:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125745:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573727.125748:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573727.125752:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125755:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573727.125758:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573727.125765:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573727.125769:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573727.125771:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573727.125775:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573727.125781:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573727.125783:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573727.125787:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788573727.125791:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573727.125794:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573727.125796:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573727.125811:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573727.125813:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573727.125814:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573727.125818:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573727.125820:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573727.125829:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393081216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573727.125837:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573727.125841:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573727.125845:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393081216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573727.125867:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573727.125872:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573727.125875:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573727.125880:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573727.125883:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573727.125888:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573727.125896:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10658
00000400:00000200:1.0:1788573727.125906:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573727.125914:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573727.125920:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573727.125928:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125934:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573727.125942:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125948:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573727.125954:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573727.125956:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573727.125958:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573727.125960:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573727.125964:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573727.125979:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573727.125985:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125990:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573727.125995:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573727.125997:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573727.126001:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573727.126004:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573727.126009:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573727.126014:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573727.126017:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573727.126022:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573727.126022:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788573727.126025:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:2.1:1788573727.126027:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000008:00100000:1.0:1788573727.126028:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573727.126032:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573727.126035:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573727.126041:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788573727.126044:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573727.126050:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573727.126053:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573727.126057:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573727.126059:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573727.126063:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573727.126069:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573727.126075:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573727.126084:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393081344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573727.126095:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573727.126099:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573727.126106:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393081344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573727.126116:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573727.126122:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573727.126128:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573727.126135:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10668)
00000800:00000200:2.1:1788573727.126544:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573727.126547:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573727.126890:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573727.126893:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:1.0:1788573727.509571:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573727.509578:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573727.509583:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573728.085533:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305331641, at 4305331648
00000100:00000010:1.0:1788573728.085553:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000020:00000001:1.0:1788573729.557534:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573729.557537:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573729.557541:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573730.581568:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573730.581572:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573730.581575:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573730.709482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573730.709489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573730.709491:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573730.709497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573730.709500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573730.709504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573730.709507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573730.709510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573730.709514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573731.733512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573731.733519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573731.733521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573731.733528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573731.733531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573731.733535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573731.733538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573731.733540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573731.733544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573732.245489:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573732.245501:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573732.245504:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573732.245508:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573732.245514:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573732.245522:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10663
00000400:00000200:1.0:1788573732.245540:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573732.245547:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573732.245550:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573732.245554:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245557:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573732.245560:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245564:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573732.245568:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573732.245570:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573732.245573:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573732.245576:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573732.245578:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573732.245580:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573732.245583:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245585:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573732.245588:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245589:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573732.245593:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573732.245596:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245599:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573732.245602:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573732.245605:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573732.245608:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573732.245611:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573732.245613:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573732.245622:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788573732.245625:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573732.245630:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573732.245637:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573732.245641:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573732.245644:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573732.245648:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573732.245656:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788573732.245663:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573732.245672:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393081472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573732.245684:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573732.245688:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573732.245693:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393081472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573732.245746:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573732.245750:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573732.245754:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573732.245756:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573732.245758:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573732.245762:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573732.245765:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10663
00000400:00000200:1.0:1788573732.245777:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573732.245784:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573732.245789:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573732.245797:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245802:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573732.245809:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245816:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573732.245823:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573732.245826:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573732.245831:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573732.245836:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573732.245841:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573732.245846:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573732.245856:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245862:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573732.245870:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245875:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573732.245880:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573732.245889:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573732.245896:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573732.245903:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573732.245911:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573732.245917:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573732.245924:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573732.245932:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573732.245938:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573732.245943:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573732.245951:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788573732.245958:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573732.245965:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573732.245970:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573732.245976:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573732.245981:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573732.245986:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573732.245996:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788573732.246003:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573732.246014:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393081600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1788573732.246031:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788573732.246037:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:2.1:1788573732.246037:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:1.0:1788573732.246043:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573732.246054:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393081600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573732.246076:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573732.246083:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573732.246094:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573732.246099:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573732.246102:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573732.246109:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573732.246116:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10663
00000400:00000200:1.0:1788573732.246129:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573732.246139:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573732.246144:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573732.246151:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.246156:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573732.246159:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.246162:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573732.246164:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573732.246167:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573732.246170:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573732.246174:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573732.246181:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573732.246185:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573732.246192:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.246197:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573732.246203:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573732.246209:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573732.246215:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573732.246224:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573732.246232:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573732.246238:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573732.246244:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573732.246251:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573732.246258:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573732.246263:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573732.246272:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573732.246276:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573732.246295:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788573732.246302:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573732.246306:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573732.246309:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573732.246314:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573732.246317:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573732.246321:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573732.246328:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788573732.246336:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573732.246347:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393081728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573732.246369:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573732.246374:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573732.246384:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393081728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573732.246432:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573732.246438:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573732.246446:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573732.246455:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10673)
00000800:00000200:2.1:1788573732.246579:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573732.246581:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788573732.246824:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573732.246826:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:1.0:1788573732.629589:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573732.629592:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573732.629594:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573732.757442:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573732.757448:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573732.757450:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573732.757455:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573732.757458:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573732.757461:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573732.757463:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573732.757465:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573732.757469:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573733.205496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305336761, at 4305336768
00000100:00000010:1.0:1788573733.205514:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000020:00000001:1.0:1788573733.653539:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573733.653542:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573733.653545:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573733.781538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573733.781544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573733.781545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573733.781549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573733.781553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573733.781556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573733.781558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573733.781560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573733.781562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573734.805489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573734.805495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573734.805497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573734.805501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573734.805504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573734.805507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573734.805509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573734.805511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573734.805514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573735.701567:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573735.701571:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573735.701575:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573735.829494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573735.829499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573735.829501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573735.829505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573735.829508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573735.829511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573735.829513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573735.829515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573735.829517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573736.725591:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573736.725594:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573736.725597:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573736.853481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573736.853489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573736.853492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573736.853497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573736.853501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573736.853504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573736.853507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573736.853509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573736.853512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573737.365480:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573737.365489:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573737.365491:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573737.365506:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573737.365510:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573737.365516:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10668
00000400:00000200:1.0:1788573737.365523:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573737.365528:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573737.365530:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573737.365533:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365535:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573737.365537:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365539:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573737.365542:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573737.365543:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573737.365546:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573737.365548:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573737.365550:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573737.365551:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573737.365553:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365554:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573737.365556:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365557:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573737.365560:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573737.365562:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365564:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573737.365566:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573737.365569:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573737.365571:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573737.365573:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573737.365575:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573737.365583:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788573737.365585:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573737.365588:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573737.365589:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573737.365592:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573737.365594:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573737.365609:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573737.365616:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788573737.365621:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573737.365628:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393081856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573737.365636:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573737.365640:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573737.365643:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393081856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573737.365661:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573737.365664:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573737.365666:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573737.365667:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573737.365669:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573737.365671:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573737.365673:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10668
00000400:00000200:1.0:1788573737.365676:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573737.365679:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573737.365681:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573737.365683:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365685:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573737.365689:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365691:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573737.365695:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573737.365697:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573737.365699:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573737.365703:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573737.365705:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573737.365709:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573737.365716:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365719:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573737.365723:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365725:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573737.365729:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573737.365732:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365736:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573737.365741:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573737.365747:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573737.365751:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573737.365755:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573737.365761:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573737.365765:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573737.365768:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573737.365773:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788573737.365776:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573737.365781:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573737.365784:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573737.365786:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573737.365787:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573737.365790:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573737.365797:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788573737.365801:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573737.365811:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393081984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573737.365821:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573737.365825:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573737.365831:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393081984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573737.365849:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573737.365853:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573737.365859:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573737.365862:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573737.365864:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573737.365870:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573737.365874:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10668
00000400:00000200:1.0:1788573737.365877:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573737.365882:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573737.365885:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573737.365888:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365891:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573737.365896:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365899:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573737.365902:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573737.365904:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573737.365906:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573737.365909:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573737.365913:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573737.365918:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573737.365924:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365929:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573737.365934:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365936:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573737.365938:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573737.365942:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573737.365947:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573737.365950:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573737.365954:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573737.365962:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573737.365964:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573737.365967:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573737.365973:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573737.365976:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573737.365984:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788573737.365987:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573737.365991:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573737.365993:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573737.365996:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573737.365998:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573737.366001:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573737.366004:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:1.0:1788573737.366006:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000800:00000200:2.1:1788573737.366008:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:1.0:1788573737.366010:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573737.366017:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393082112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573737.366026:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573737.366029:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573737.366037:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393082112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573737.366048:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573737.366051:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573737.366056:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573737.366061:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10678)
00000800:00000200:2.1:1788573737.366359:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573737.366361:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788573737.366581:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573737.366583:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:1.0:1788573737.877500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573737.877508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573737.877512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573737.877518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573737.877522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573737.877527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573737.877531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573737.877534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573737.877538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573738.325484:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305341881, at 4305341888
00000100:00000010:1.0:1788573738.325503:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000020:00000001:1.0:1788573738.773513:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573738.773516:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573738.773519:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573738.901461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573738.901468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573738.901470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573738.901474:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573738.901477:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573738.901480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573738.901482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573738.901495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573738.901497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573739.797652:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573739.797658:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573739.797662:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573739.925518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573739.925527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573739.925529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573739.925534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573739.925538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573739.925542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573739.925579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573739.925582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573739.925585:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573740.949459:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573740.949476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573740.949479:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573740.949485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573740.949489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573740.949494:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573740.949497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573740.949500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573740.949504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573741.973487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573741.973492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573741.973494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573741.973499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573741.973501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573741.973528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573741.973531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573741.973533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573741.973536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573742.486108:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573742.486112:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788573742.486632:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573742.486635:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788573742.486966:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573742.486969:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:1.0:1788573742.869570:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573742.869574:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573742.869576:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573742.997472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573742.997480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573742.997482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573742.997519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573742.997523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573742.997527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573742.997530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573742.997534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573742.997537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573743.445445:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305347001, at 4305347008
00000100:00000010:1.0:1788573743.445458:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000100:00000001:1.0:1788573744.021511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573744.021518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573744.021524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573744.021530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573744.021533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573744.021536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573744.021538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573744.021540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573744.021542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573745.045477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573745.045488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573745.045490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573745.045495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573745.045499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573745.045502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573745.045504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573745.045506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573745.045508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573745.941589:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573745.941594:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573745.941598:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573746.069514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573746.069520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573746.069521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573746.069525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573746.069528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573746.069531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573746.069533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573746.069535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573746.069538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573747.093560:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573747.093570:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573747.093573:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573747.093581:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573747.093586:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573747.093591:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573747.093595:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573747.093598:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573747.093602:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573747.606099:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573747.606103:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1788573747.606460:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573747.606462:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1788573747.606660:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573747.606662:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00000001:1.0:1788573748.117473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573748.117481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573748.117483:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573748.117488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573748.117491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573748.117495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573748.117497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573748.117498:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573748.117501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573748.565490:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305352121, at 4305352128
00000100:00000010:1.0:1788573748.565508:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f00.
00000020:00000001:1.0:1788573749.013532:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573749.013535:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573749.013538:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573749.141541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573749.141550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573749.141552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573749.141558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573749.141561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573749.141565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573749.141568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573749.141571:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573749.141573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573750.165485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573750.165496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573750.165499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573750.165506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573750.165511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573750.165516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573750.165520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573750.165523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573750.165527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573751.189599:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573751.189607:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573751.189610:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573751.189615:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573751.189619:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573751.189623:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573751.189625:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573751.189628:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573751.189631:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573752.085642:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573752.085647:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573752.085651:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573752.725551:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573752.725562:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573752.725565:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573752.725569:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573752.725574:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573752.725582:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10683
00000400:00000200:1.0:1788573752.725598:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573752.725605:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573752.725608:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573752.725611:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725614:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573752.725616:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725618:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573752.725622:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573752.725623:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573752.725626:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573752.725629:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573752.725631:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573752.725633:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573752.725636:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725637:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573752.725639:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725640:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573752.725643:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573752.725645:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725647:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573752.725650:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573752.725653:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573752.725656:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573752.725658:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573752.725660:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573752.725671:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788573752.725674:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573752.725678:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573752.725679:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573752.725682:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573752.725685:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573752.725690:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573752.725698:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573752.725705:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573752.725715:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393083008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573752.725724:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573752.725729:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573752.725733:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393083008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573752.725755:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573752.725758:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573752.725761:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573752.725762:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573752.725764:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573752.725766:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573752.725768:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10683
00000400:00000200:1.0:1788573752.725771:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573752.725773:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573752.725775:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573752.725777:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725779:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573752.725781:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725783:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573752.725785:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573752.725787:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573752.725788:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573752.725790:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573752.725792:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573752.725793:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573752.725797:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725799:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573752.725803:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725805:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573752.725809:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573752.725814:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725817:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573752.725818:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573752.725824:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573752.725829:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573752.725833:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573752.725839:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573752.725844:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573752.725847:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573752.725853:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788573752.725859:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573752.725862:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573752.725865:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573752.725869:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573752.725873:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573752.725878:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573752.725886:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573752.725893:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573752.725905:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393083136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573752.725916:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573752.725919:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573752.725925:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393083136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573752.725933:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573752.725939:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573752.725944:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573752.725947:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573752.725948:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573752.725953:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573752.725958:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10683
00000400:00000200:1.0:1788573752.725965:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573752.725972:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573752.725975:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573752.725980:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725984:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573752.725986:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.725989:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573752.725993:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573752.725995:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573752.725997:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573752.725998:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573752.726000:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573752.726004:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573752.726010:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.726012:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573752.726016:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573752.726021:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573752.726024:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573752.726028:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573752.726031:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573752.726033:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573752.726037:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573752.726040:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573752.726044:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573752.726048:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573752.726052:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573752.726058:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573752.726063:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788573752.726066:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573752.726069:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573752.726072:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573752.726076:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573752.726078:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573752.726081:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573752.726085:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573752.726090:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573752.726096:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393083264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573752.726106:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573752.726110:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573752.726115:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393083264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573752.726123:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573752.726125:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573752.726131:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:0.1:1788573752.726131:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573752.726137:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000100:00000040:1.0:1788573752.726142:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10693)
00000800:00000200:0.1:1788573752.726630:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573752.726632:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1788573752.726920:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573752.726923:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000100:00000040:1.0:1788573753.685532:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305357241, at 4305357248
00000100:00000010:1.0:1788573753.685553:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7600.
00000800:00000200:1.0:1788573753.749747:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573755.157533:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573755.157536:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573755.157539:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573757.846022:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573757.846026:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573757.846483:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573757.846485:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573757.846769:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573757.846772:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788573758.229601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573758.229604:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573758.229607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573758.805467:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305362361, at 4305362368
00000100:00000010:1.0:1788573758.805486:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7840.
00000020:00000001:1.0:1788573761.301557:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573761.301561:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573761.301564:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573762.966130:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573762.966135:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573762.966656:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573762.966658:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573762.966913:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573762.966915:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:1.0:1788573763.925491:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305367481, at 4305367488
00000100:00000010:1.0:1788573763.925504:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d80.
00000020:00000001:1.0:1788573764.373584:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573764.373592:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573764.373605:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573766.549567:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573766.549573:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573766.549575:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573766.549580:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573766.549583:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573766.549586:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573766.549588:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573766.549590:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573766.549592:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573767.445687:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573767.445691:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573767.445694:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573768.086307:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573768.086312:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573768.086826:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573768.086828:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573768.087039:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573768.087041:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:1.0:1788573769.045497:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305372602, at 4305372608
00000100:00000010:1.0:1788573769.045510:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7660.
00000020:00000001:1.0:1788573770.517592:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573770.517595:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573770.517598:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573773.205516:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573773.205527:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573773.205530:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573773.205534:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573773.205539:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573773.205547:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10704
00000400:00000200:1.0:1788573773.205566:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573773.205573:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573773.205575:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573773.205578:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205581:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573773.205583:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205585:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573773.205588:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573773.205590:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573773.205592:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573773.205595:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573773.205597:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573773.205599:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573773.205604:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205606:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573773.205607:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205609:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573773.205611:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573773.205614:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205616:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573773.205618:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573773.205621:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573773.205623:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573773.205624:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573773.205626:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573773.205634:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788573773.205636:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573773.205639:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573773.205640:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573773.205642:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573773.205645:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573773.205648:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573773.205654:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573773.205660:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573773.205670:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393084544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573773.205681:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573773.205686:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573773.205692:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393084544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573773.205718:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573773.205721:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573773.205724:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573773.205725:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573773.205726:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573773.205728:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573773.205731:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10704
00000400:00000200:1.0:1788573773.205734:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573773.205736:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573773.205738:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573773.205740:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205741:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573773.205743:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205744:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573773.205746:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573773.205747:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573773.205748:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573773.205749:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573773.205751:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573773.205752:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573773.205754:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205756:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573773.205758:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205759:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573773.205762:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573773.205764:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205766:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573773.205768:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573773.205773:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573773.205776:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573773.205778:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573773.205783:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573773.205786:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573773.205788:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573773.205792:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788573773.205796:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573773.205800:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573773.205803:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573773.205806:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573773.205809:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573773.205812:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573773.205817:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788573773.205824:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573773.205831:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393084672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573773.205843:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573773.205847:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573773.205853:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393084672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573773.205865:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573773.205868:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573773.205874:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573773.205876:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573773.205879:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573773.205880:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573773.205884:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10704
00000400:00000200:1.0:1788573773.205905:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573773.205909:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573773.205911:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573773.205913:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205916:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573773.205920:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205922:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573773.205924:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573773.205925:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573773.205928:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573773.205936:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573773.205939:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573773.205942:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573773.205947:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205952:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573773.205955:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205959:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573773.205965:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573773.205971:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573773.205977:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573773.205982:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573773.205987:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573773.205992:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573773.205997:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573773.206002:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573773.206006:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573773.206008:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573773.206016:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788573773.206021:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573773.206027:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573773.206028:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573773.206029:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573773.206031:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573773.206035:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573773.206041:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788573773.206046:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573773.206054:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393084800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573773.206065:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573773.206069:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573773.206075:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393084800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573773.206089:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573773.206096:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573773.206104:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573773.206113:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10714)
00000800:00000200:0.1:1788573773.206124:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573773.206127:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788573773.206460:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573773.206463:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788573773.206674:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573773.206676:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:1.0:1788573773.589628:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573773.589634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573773.589638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573774.165521:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305377721, at 4305377728
00000100:00000010:1.0:1788573774.165541:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d70c0.
00000020:00000001:1.0:1788573776.661576:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573776.661581:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573776.661585:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573778.326013:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573778.326245:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573778.326386:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788573779.285508:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305382841, at 4305382848
00000100:00000010:1.0:1788573779.285535:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ae0.
00000020:00000001:1.0:1788573779.733627:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573779.733633:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573779.733638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573780.885541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573780.885546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573780.885548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573780.885553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573780.885556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573780.885559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573780.885561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573780.885564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573780.885566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788573781.397641:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788573781.397687:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788573781.397695:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573781.397700:0:2349:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788573781.397704:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573781.397708:0:2349:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788573781.397714:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573781.397718:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:1.0:1788573781.397725:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573781.397760:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573781.397768:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4990
00000400:00000010:1.0:1788573781.397770:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4990.
00000100:00000001:1.0:1788573781.397776:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573781.397786:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0a80 x1875444393084928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573781.397812:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573781.397817:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393084928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573781.397825:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573781.397827:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573781.397838:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573781.397841:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9900.
00000400:00000200:1.0:1788573781.397845:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573781.397863:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573781.397867:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4b28
00000400:00000010:1.0:1788573781.397869:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4b28.
00000100:00000001:1.0:1788573781.397873:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573781.397881:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2300 x1875444393085056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573781.397899:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573781.397907:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393085056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573781.397925:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573781.397927:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573781.397935:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573781.397941:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9400.
00000400:00000200:1.0:1788573781.397949:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573781.397956:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573781.397965:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4aa0
00000400:00000010:1.0:1788573781.397969:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4aa0.
00000100:00000001:1.0:1788573781.397975:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573781.397985:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0380 x1875444393085184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573781.398004:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573781.398012:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393085184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573781.398026:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573781.398030:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788573781.909451:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573781.909458:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573781.909460:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573781.909464:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573781.909467:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573781.909478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573781.909480:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573781.909483:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573781.909485:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573782.805647:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573782.805653:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573782.805658:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573782.933477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573782.933483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573782.933485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573782.933489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573782.933492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573782.933495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573782.933497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573782.933500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573782.933502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573783.446443:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573783.446447:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573783.446889:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573783.446891:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573783.447218:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573783.447219:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:1.0:1788573783.957531:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573783.957537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573783.957539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573783.957543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573783.957546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573783.957549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573783.957552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573783.957554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573783.957556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573784.405503:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305387962, at 4305387968
00000100:00000010:1.0:1788573784.405527:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000100:00000001:1.0:1788573784.981477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573784.981483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573784.981484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573784.981489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573784.981492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573784.981495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573784.981497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573784.981500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573784.981502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573785.877590:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573785.877598:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573785.877604:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573786.005463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573786.005469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573786.005471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573786.005475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573786.005478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573786.005480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573786.005482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573786.005484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573786.005486:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573787.029489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573787.029500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573787.029503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573787.029511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573787.029515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573787.029520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573787.029524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573787.029528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573787.029531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573788.053505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573788.053514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573788.053517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573788.053523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573788.053526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573788.053530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573788.053533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573788.053536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573788.053541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573788.565516:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573788.565527:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573788.565530:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573788.565535:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573788.565540:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573788.565550:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10719
00000400:00000200:1.0:1788573788.565569:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573788.565578:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573788.565582:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573788.565587:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565591:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573788.565594:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565597:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573788.565602:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573788.565617:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573788.565620:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573788.565623:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573788.565625:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573788.565627:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573788.565630:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565632:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573788.565633:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565635:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573788.565638:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573788.565641:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565643:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573788.565645:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573788.565649:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573788.565651:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573788.565653:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573788.565655:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573788.565665:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788573788.565668:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573788.565672:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573788.565673:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573788.565676:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573788.565679:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573788.565685:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573788.565695:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788573788.565704:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573788.565717:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393085696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573788.565732:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573788.565737:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573788.565742:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393085696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573788.565764:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573788.565769:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573788.565773:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573788.565775:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573788.565777:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573788.565781:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573788.565785:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10719
00000400:00000200:1.0:1788573788.565790:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573788.565795:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573788.565798:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573788.565803:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565806:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573788.565809:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565813:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573788.565816:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573788.565818:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573788.565821:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573788.565824:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573788.565829:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573788.565833:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573788.565838:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565844:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573788.565848:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565851:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573788.565856:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573788.565862:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573788.565868:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573788.565870:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573788.565877:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573788.565883:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573788.565887:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573788.565892:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573788.565899:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573788.565903:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573788.565910:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788573788.565916:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573788.565922:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573788.565926:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573788.565930:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573788.565933:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573788.565937:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573788.565947:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573788.565953:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573788.565970:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393085824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573788.565984:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573788.565986:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573788.565990:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393085824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573788.566002:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573788.566005:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573788.566009:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573788.566011:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573788.566015:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573788.566019:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573788.566025:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10719
00000400:00000200:1.0:1788573788.566034:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573788.566041:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573788.566045:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573788.566050:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.566052:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573788.566054:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.566055:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573788.566057:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573788.566058:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573788.566059:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573788.566061:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573788.566062:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573788.566064:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573788.566071:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.566073:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573788.566078:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573788.566081:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573788.566085:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573788.566091:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573788.566094:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573788.566099:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573788.566103:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573788.566108:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573788.566113:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573788.566118:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:2.1:1788573788.566119:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000008:00000001:1.0:1788573788.566125:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573788.566125:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788573788.566130:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573788.566138:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788573788.566143:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573788.566148:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573788.566152:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573788.566155:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573788.566157:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573788.566159:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573788.566164:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788573788.566169:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573788.566177:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393085952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573788.566195:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573788.566201:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573788.566212:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393085952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573788.566232:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573788.566239:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573788.566243:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573788.566250:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10729)
00000800:00000200:2.1:1788573788.566543:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573788.566545:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573788.566773:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573788.566775:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:1.0:1788573788.949621:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573788.949625:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573788.949629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573789.077461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573789.077469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573789.077471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573789.077477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573789.077481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573789.077486:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573789.077488:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573789.077491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573789.077494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573789.525448:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305393081, at 4305393088
00000100:00000010:1.0:1788573789.525464:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000100:00000001:1.0:1788573790.101486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573790.101493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573790.101495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573790.101500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573790.101503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573790.101506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573790.101508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573790.101510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573790.101512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573791.125493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573791.125499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573791.125500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573791.125504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573791.125506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573791.125510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573791.125512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573791.125514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573791.125515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788573791.637791:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573792.021629:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573792.021634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573792.021638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573792.149517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573792.149523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573792.149525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573792.149529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573792.149533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573792.149536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573792.149538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573792.149559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573792.149563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573793.173485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573793.173491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573793.173493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573793.173497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573793.173500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573793.173503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573793.173505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573793.173507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573793.173509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573793.685534:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573793.685547:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573793.685550:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573793.685554:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573793.685559:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573793.685566:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10724
00000400:00000200:1.0:1788573793.685576:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573793.685583:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573793.685586:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573793.685588:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685591:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573793.685593:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685596:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573793.685599:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573793.685600:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573793.685603:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573793.685606:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573793.685608:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573793.685610:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573793.685612:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685613:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573793.685614:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685616:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573793.685619:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573793.685621:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685624:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573793.685626:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573793.685629:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573793.685632:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573793.685634:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573793.685636:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573793.685647:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788573793.685650:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573793.685655:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573793.685656:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573793.685659:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573793.685663:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573793.685667:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573793.685677:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788573793.685686:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573793.685699:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393086080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573793.685711:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573793.685717:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573793.685723:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393086080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573793.685751:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573793.685754:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573793.685757:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573793.685758:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573793.685759:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573793.685763:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573793.685766:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10724
00000400:00000200:1.0:1788573793.685770:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573793.685773:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573793.685775:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573793.685778:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685779:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573793.685790:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685798:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573793.685805:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573793.685807:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573793.685814:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573793.685818:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573793.685820:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573793.685822:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573793.685827:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685829:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573793.685831:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685837:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573793.685843:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573793.685850:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573793.685852:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573793.685861:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573793.685872:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573793.685882:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573793.685891:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573793.685902:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573793.685913:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573793.685919:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573793.685925:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788573793.685930:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573793.685932:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573793.685933:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573793.685936:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573793.685943:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573793.685948:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573793.685955:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788573793.685962:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573793.685972:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393086208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573793.685982:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573793.685991:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573793.685999:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393086208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573793.686021:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573793.686031:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573793.686039:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573793.686044:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573793.686047:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573793.686057:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573793.686062:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10724
00000400:00000200:1.0:1788573793.686069:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573793.686072:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573793.686074:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573793.686082:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.686086:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573793.686091:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.686094:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573793.686097:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573793.686100:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573793.686104:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573793.686109:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573793.686112:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573793.686118:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573793.686128:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.686135:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573793.686142:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573793.686151:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573793.686159:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573793.686166:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573793.686178:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573793.686181:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573793.686191:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573793.686195:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573793.686202:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573793.686207:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573793.686218:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573793.686224:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573793.686232:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788573793.686236:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573793.686244:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573793.686248:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573793.686251:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788573793.686252:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573793.686255:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573793.686257:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573793.686262:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788573793.686265:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573793.686273:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00100000:1.0:1788573793.686276:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393086336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573793.686292:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573793.686297:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573793.686304:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393086336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573793.686319:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573793.686324:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573793.686334:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573793.686341:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10734)
00000800:00000200:2.1:1788573793.686685:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573793.686687:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573793.686895:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573793.686897:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:1.0:1788573794.197545:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573794.197553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573794.197555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573794.197560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573794.197564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573794.197568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573794.197571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573794.197574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573794.197582:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573794.645494:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305398202, at 4305398208
00000100:00000010:1.0:1788573794.645526:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000020:00000001:1.0:1788573795.093534:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573795.093537:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573795.093540:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573796.245523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573796.245529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573796.245532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573796.245537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573796.245541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573796.245545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573796.245548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573796.245552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573796.245556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573797.269525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573797.269532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573797.269534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573797.269538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573797.269541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573797.269544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573797.269546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573797.269548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573797.269550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573798.165589:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573798.165594:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573798.165597:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573798.293501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573798.293506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573798.293508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573798.293513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573798.293517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573798.293521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573798.293530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573798.293533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573798.293538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573798.806021:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573798.806027:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788573798.806385:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573798.806387:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788573798.806607:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573798.806609:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:1.0:1788573799.317477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573799.317483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573799.317485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573799.317489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573799.317498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573799.317501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573799.317503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573799.317505:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573799.317508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573799.765494:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305403321, at 4305403328
00000100:00000010:1.0:1788573799.765517:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000100:00000001:1.0:1788573800.341495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573800.341501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573800.341502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573800.341507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573800.341509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573800.341513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573800.341515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573800.341517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573800.341519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573801.237595:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573801.237600:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573801.237604:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573801.365497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573801.365504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573801.365506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573801.365510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573801.365517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573801.365521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573801.365523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573801.365525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573801.365527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573802.389498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573802.389506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573802.389508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573802.389513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573802.389517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573802.389520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573802.389523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573802.389525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573802.389527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573803.413516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573803.413523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573803.413524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573803.413529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573803.413531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573803.413535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573803.413537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573803.413539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573803.413541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573803.926203:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573803.926208:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788573803.926754:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573803.926756:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788573803.926976:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573803.926978:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:1.0:1788573804.309628:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573804.309633:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573804.309638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573804.437482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573804.437491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573804.437493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573804.437500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573804.437504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573804.437508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573804.437511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573804.437514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573804.437516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573804.885470:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305408441, at 4305408448
00000100:00000010:1.0:1788573804.885488:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d79c0.
00000100:00000001:1.0:1788573805.461555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573805.461564:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573805.461566:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573805.461571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573805.461575:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573805.461579:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573805.461581:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573805.461584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573805.461587:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573806.485494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573806.485503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573806.485505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573806.485510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573806.485514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573806.485518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573806.485521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573806.485524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573806.485526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573807.381507:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573807.381511:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573807.381514:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573807.509505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573807.509513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573807.509516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573807.509521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573807.509525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573807.509530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573807.509533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573807.509535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573807.509538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573808.533459:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573808.533466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573808.533469:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573808.533474:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573808.533478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573808.533482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573808.533485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573808.533488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573808.533491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573809.046060:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573809.046068:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573809.046473:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573809.046475:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573809.046704:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573809.046706:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:1.0:1788573809.557489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573809.557498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573809.557501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573809.557507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573809.557510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573809.557513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573809.557516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573809.557519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573809.557523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573810.005464:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305413561, at 4305413568
00000100:00000010:1.0:1788573810.005472:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7360.
00000020:00000001:1.0:1788573810.453604:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573810.453607:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573810.453610:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573810.581452:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573810.581458:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573810.581460:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573810.581465:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573810.581468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573810.581471:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573810.581473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573810.581474:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573810.581477:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573811.605523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573811.605531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573811.605534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573811.605540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573811.605544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573811.605549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573811.605551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573811.605554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573811.605557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573812.629482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573812.629490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573812.629492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573812.629499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573812.629503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573812.629507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573812.629510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573812.629514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573812.629517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573813.525577:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573813.525583:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573813.525587:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573813.653540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573813.653549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573813.653551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573813.653556:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573813.653560:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573813.653564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573813.653567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573813.653570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573813.653573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573814.166169:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573814.166175:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573814.166687:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573814.166690:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573814.166947:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573814.166948:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1788573814.677518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573814.677526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573814.677529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573814.677535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573814.677539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573814.677543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573814.677546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573814.677549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573814.677552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573815.125504:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305418681, at 4305418688
00000100:00000010:1.0:1788573815.125525:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d72a0.
00000100:00000001:1.0:1788573815.701533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573815.701539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573815.701541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573815.701545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573815.701547:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573815.701550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573815.701552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573815.701554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573815.701557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573816.597581:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573816.597584:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573816.597587:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573816.725476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573816.725482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573816.725484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573816.725489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573816.725493:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573816.725496:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573816.725501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573816.725504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573816.725510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573817.749478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573817.749484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573817.749486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573817.749490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573817.749493:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573817.749496:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573817.749498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573817.749500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573817.749502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573818.773478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573818.773485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573818.773488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573818.773494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573818.773498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573818.773502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573818.773505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573818.773507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573818.773511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573819.286305:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573819.286312:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573819.286785:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573819.286789:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573819.287097:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573819.287098:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788573819.669525:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573819.669529:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573819.669532:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573819.797475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573819.797482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573819.797483:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573819.797487:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573819.797490:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573819.797494:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573819.797496:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573819.797498:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573819.797500:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573820.245492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305423802, at 4305423808
00000100:00000010:1.0:1788573820.245509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7420.
00000100:00000001:1.0:1788573820.821476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573820.821482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573820.821484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573820.821488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573820.821491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573820.821495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573820.821497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573820.821499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573820.821501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573821.845502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573821.845508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573821.845510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573821.845514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573821.845517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573821.845521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573821.845523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573821.845525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573821.845528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573822.741527:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573822.741533:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573822.741536:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573822.869525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573822.869532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573822.869534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573822.869539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573822.869542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573822.869545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573822.869547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573822.869549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573822.869551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573823.893489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573823.893497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573823.893500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573823.893506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573823.893510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573823.893513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573823.893516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573823.893519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573823.893532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573824.405998:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573824.406004:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573824.406344:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573824.406345:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573824.406588:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573824.406589:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:1.0:1788573824.917552:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573824.917559:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573824.917560:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573824.917565:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573824.917568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573824.917571:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573824.917573:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573824.917583:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573824.917585:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573825.365499:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305428921, at 4305428928
00000100:00000010:1.0:1788573825.365514:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d74e0.
00000020:00000001:1.0:1788573825.813591:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573825.813594:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573825.813597:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573825.941522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573825.941529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573825.941531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573825.941536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573825.941539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573825.941542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573825.941544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573825.941546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573825.941548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573826.965513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573826.965518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573826.965520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573826.965524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573826.965527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573826.965530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573826.965532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573826.965534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573826.965536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573827.989458:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573827.989464:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573827.989466:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573827.989470:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573827.989472:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573827.989476:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573827.989479:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573827.989481:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573827.989483:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573828.885570:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573828.885574:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573828.885577:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573829.013482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573829.013490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573829.013492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573829.013498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573829.013502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573829.013506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573829.013509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573829.013512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573829.013515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573829.526003:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573829.526009:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573829.526361:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573829.526363:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573829.526664:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573829.526665:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788573830.037442:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573830.037448:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573830.037449:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573830.037454:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573830.037457:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573830.037460:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573830.037462:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573830.037464:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573830.037466:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573830.485492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305434041, at 4305434048
00000100:00000010:1.0:1788573830.485510:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7000.
00000800:00000200:1.0:1788573830.549665:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000001:1.0:1788573831.061509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573831.061518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573831.061521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573831.061527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573831.061531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573831.061534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573831.061536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573831.061538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573831.061540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573831.957582:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573831.957587:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573831.957591:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573832.085527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573832.085533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573832.085535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573832.085540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573832.085542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573832.085546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573832.085548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573832.085550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573832.085552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573833.109508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573833.109516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573833.109518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573833.109524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573833.109528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573833.109532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573833.109535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573833.109537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573833.109540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573834.133504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573834.133510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573834.133512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573834.133517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573834.133520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573834.133523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573834.133525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573834.133527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573834.133529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573834.646138:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573834.646142:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573834.646545:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573834.646547:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573834.646794:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573834.646796:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:1.0:1788573835.029527:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573835.029531:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573835.029533:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573835.157486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573835.157492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573835.157493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573835.157497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573835.157500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573835.157503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573835.157506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573835.157508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573835.157517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573835.605507:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305439161, at 4305439168
00000100:00000010:1.0:1788573835.605521:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7000.
00000100:00000001:1.0:1788573836.181463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573836.181469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573836.181472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573836.181476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573836.181479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573836.181483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573836.181485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573836.181487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573836.181489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573837.205514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573837.205528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573837.205530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573837.205535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573837.205538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573837.205541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573837.205543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573837.205546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573837.205548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573838.101580:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573838.101583:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573838.101586:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573838.229479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573838.229487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573838.229490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573838.229495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573838.229499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573838.229504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573838.229507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573838.229510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573838.229513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573839.253463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573839.253470:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573839.253473:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573839.253478:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573839.253481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573839.253484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573839.253486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573839.253488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573839.253490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573839.766100:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573839.766105:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788573839.766530:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573839.766531:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788573839.766774:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573839.766776:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:1.0:1788573840.277468:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573840.277476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573840.277478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573840.277484:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573840.277488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573840.277492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573840.277495:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573840.277497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573840.277499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573840.725508:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305444281, at 4305444288
00000100:00000010:1.0:1788573840.725527:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d74e0.
00000020:00000001:1.0:1788573841.173629:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573841.173635:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573841.173638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573841.301536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573841.301545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573841.301547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573841.301553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573841.301556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573841.301561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573841.301563:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573841.301566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573841.301569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573842.325465:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573842.325471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573842.325473:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573842.325477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573842.325480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573842.325483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573842.325485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573842.325487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573842.325489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573843.349521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573843.349530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573843.349532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573843.349538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573843.349542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573843.349546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573843.349549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573843.349552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573843.349555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573844.245589:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573844.245595:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573844.245598:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573844.373557:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573844.373568:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573844.373571:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573844.373578:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573844.373583:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573844.373589:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573844.373592:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573844.373597:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573844.373601:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573844.886049:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573844.886280:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573844.886443:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788573845.397462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573845.397469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573845.397472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573845.397476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573845.397480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573845.397483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573845.397485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573845.397488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573845.397490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573845.845478:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305449401, at 4305449408
00000100:00000010:1.0:1788573845.845499:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7420.
00000100:00000001:1.0:1788573846.421502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573846.421508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573846.421510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573846.421515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573846.421518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573846.421522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573846.421524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573846.421534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573846.421537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573847.317571:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573847.317597:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573847.317600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573847.445481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573847.445488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573847.445490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573847.445496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573847.445499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573847.445503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573847.445506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573847.445508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573847.445511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573848.469505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573848.469513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573848.469515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573848.469519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573848.469522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573848.469526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573848.469528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573848.469530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573848.469532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573849.493475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573849.493482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573849.493485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573849.493491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573849.493495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573849.493499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573849.493502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573849.493504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573849.493507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573850.006329:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573850.006333:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573850.006770:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573850.006772:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573850.007069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573850.007072:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1788573850.389555:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573850.389558:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573850.389561:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573850.517521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573850.517527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573850.517528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573850.517533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573850.517536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573850.517539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573850.517541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573850.517543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573850.517545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573850.965489:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305454522, at 4305454528
00000100:00000010:1.0:1788573850.965519:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d72a0.
00000100:00000001:1.0:1788573851.541463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573851.541471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573851.541474:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573851.541480:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573851.541484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573851.541499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573851.541502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573851.541504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573851.541507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573852.565483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573852.565490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573852.565491:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573852.565496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573852.565500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573852.565503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573852.565505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573852.565507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573852.565510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573853.461615:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573853.461620:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573853.461624:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573853.589489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573853.589495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573853.589497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573853.589502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573853.589505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573853.589507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573853.589509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573853.589511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573853.589513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573854.613510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573854.613525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573854.613527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573854.613532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573854.613535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573854.613539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573854.613541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573854.613544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573854.613546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573855.126151:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573855.126154:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573855.126646:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573855.126649:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573855.126947:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573855.126949:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788573855.637496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573855.637502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573855.637505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573855.637508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573855.637511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573855.637514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573855.637516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573855.637519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573855.637522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573856.085488:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305459641, at 4305459648
00000100:00000010:1.0:1788573856.085506:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000020:00000001:1.0:1788573856.533532:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573856.533537:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573856.533541:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573856.661499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573856.661507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573856.661509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573856.661515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573856.661519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573856.661523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573856.661526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573856.661528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573856.661531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573857.685469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573857.685475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573857.685477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573857.685481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573857.685486:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573857.685490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573857.685492:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573857.685495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573857.685498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573858.709506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573858.709513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573858.709516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573858.709522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573858.709525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573858.709530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573858.709533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573858.709536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573858.709539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573859.605559:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573859.605563:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573859.605566:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573859.733464:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573859.733470:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573859.733472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573859.733476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573859.733479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573859.733482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573859.733484:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573859.733486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573859.733489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573860.245968:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573860.245973:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573860.246450:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573860.246452:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788573860.246715:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573860.246717:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:1.0:1788573860.757501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573860.757507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573860.757509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573860.757514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573860.757517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573860.757520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573860.757522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573860.757524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573860.757527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573861.205488:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305464761, at 4305464768
00000100:00000010:1.0:1788573861.205501:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000100:00000001:1.0:1788573861.781483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573861.781491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573861.781494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573861.781500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573861.781504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573861.781507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573861.781510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573861.781513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573861.781516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573862.677562:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573862.677566:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573862.677570:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573862.805477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573862.805484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573862.805487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573862.805493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573862.805496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573862.805500:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573862.805503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573862.805506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573862.805509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573863.829501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573863.829508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573863.829510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573863.829515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573863.829519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573863.829523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573863.829526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573863.829529:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573863.829532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573864.853476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573864.853482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573864.853484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573864.853488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573864.853491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573864.853495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573864.853497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573864.853498:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573864.853500:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788573865.366033:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573865.366039:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788573865.366384:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573865.366386:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788573865.366613:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573865.366614:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:1.0:1788573865.749550:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573865.749554:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573865.749558:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573865.877509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573865.877516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573865.877518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573865.877524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573865.877527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573865.877531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573865.877534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573865.877537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573865.877540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573866.325500:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305469881, at 4305469888
00000100:00000010:1.0:1788573866.325524:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000800:00000200:1.0:1788573868.437650:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788573868.821593:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573868.821597:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573868.821600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788573870.486300:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573870.486308:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788573870.486945:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573870.486947:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788573870.487250:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573870.487253:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:1.0:1788573871.445551:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305475002, at 4305475008
00000100:00000010:1.0:1788573871.445576:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00000020:00000001:1.0:1788573871.893593:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573871.893597:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573871.893600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788573874.965639:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573874.965660:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573874.965664:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573875.606139:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573875.606146:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788573875.606635:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573875.606637:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788573875.606849:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573875.606850:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000040:1.0:1788573876.565487:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305480121, at 4305480128
00000100:00000010:1.0:1788573876.565504:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7360.
00000020:00000001:1.0:1788573878.037544:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573878.037547:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573878.037550:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573880.726194:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573880.726198:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1788573880.726606:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573880.726608:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1788573880.726826:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573880.726828:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:1.0:1788573881.109631:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573881.109636:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573881.109646:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573881.237522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573881.237535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573881.237538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573881.237544:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573881.237548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573881.237551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573881.237555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573881.237557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573881.237561:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573881.685496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305485241, at 4305485248
00000100:00000010:1.0:1788573881.685511:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d79c0.
00000100:00000001:1.0:1788573882.261518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573882.261527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573882.261529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573882.261534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573882.261538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573882.261542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573882.261545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573882.261548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573882.261551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573883.285497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573883.285503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573883.285505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573883.285515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573883.285518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573883.285521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573883.285523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573883.285525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573883.285527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573884.181601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573884.181605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573884.181607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573884.309485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573884.309491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573884.309493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573884.309497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573884.309500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573884.309503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573884.309505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573884.309507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573884.310714:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573885.333513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573885.333519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573885.333521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573885.333525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573885.333528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573885.333532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573885.333534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573885.333536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573885.333539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573885.846080:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573885.846083:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573885.846473:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573885.846475:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573885.846745:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573885.846748:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1788573886.357463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573886.357470:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573886.357472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573886.357476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573886.357479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573886.357482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573886.357484:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573886.357486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573886.357488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573886.805469:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305490361, at 4305490368
00000100:00000010:1.0:1788573886.805486:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ae0.
00000020:00000001:1.0:1788573887.253539:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573887.253543:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573887.253547:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573887.381505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573887.381511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573887.381512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573887.381516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573887.381519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573887.381523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573887.381525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573887.381528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573887.381529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573890.325596:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573890.325602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573890.325606:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788573890.965524:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573890.965536:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573890.965539:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573890.965544:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573890.965550:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573890.965558:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10822
00000400:00000200:1.0:1788573890.965569:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573890.965576:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573890.965579:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573890.965582:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965585:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573890.965587:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965589:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573890.965593:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573890.965595:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573890.965597:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573890.965600:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573890.965602:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573890.965603:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573890.965605:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965607:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573890.965608:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965610:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573890.965612:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573890.965615:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965618:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573890.965621:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573890.965624:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573890.965627:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573890.965628:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573890.965631:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573890.965642:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788573890.965645:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573890.965648:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573890.965650:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573890.965653:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573890.965656:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573890.965661:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573890.965669:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573890.965676:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573890.965686:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393093376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573890.965725:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573890.965729:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573890.965734:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393093376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573890.965755:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573890.965757:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573890.965760:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573890.965762:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573890.965763:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573890.965766:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573890.965768:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10822
00000400:00000200:1.0:1788573890.965771:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573890.965773:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573890.965775:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573890.965777:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965779:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573890.965781:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965783:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573890.965788:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573890.965789:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573890.965791:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573890.965793:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573890.965795:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573890.965797:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573890.965799:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965800:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573890.965801:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965803:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573890.965804:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573890.965806:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965808:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573890.965809:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573890.965816:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573890.965821:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573890.965823:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573890.965829:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573890.965832:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573890.965836:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573890.965839:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788573890.965841:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573890.965843:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573890.965844:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573890.965848:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573890.965850:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573890.965853:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573890.965858:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788573890.965864:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573890.965874:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393093504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573890.965883:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573890.965886:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573890.965892:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393093504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573890.965902:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573890.965906:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573890.965912:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573890.965915:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573890.965916:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573890.965919:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573890.965922:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10822
00000400:00000200:1.0:1788573890.965929:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573890.965932:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573890.965934:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573890.965940:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965943:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573890.965948:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965950:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573890.965952:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573890.965953:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573890.965955:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573890.965958:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573890.965960:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573890.965961:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573890.965964:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965969:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573890.965972:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965976:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573890.965979:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573890.965984:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573890.965991:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573890.965994:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573890.965998:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573890.966001:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573890.966005:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573890.966009:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573890.966012:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573890.966017:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573890.966021:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788573890.966024:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573890.966028:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573890.966030:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573890.966033:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573890.966034:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573890.966035:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573890.966039:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573890.966045:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573890.966053:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393093632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573890.966061:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573890.966064:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573890.966071:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393093632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573890.966085:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573890.966090:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573890.966099:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573890.966104:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10832)
00000800:00000200:0.1:1788573890.966142:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573890.966145:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573890.966580:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573890.966582:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573890.966879:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573890.966881:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:1.0:1788573891.925476:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305495481, at 4305495488
00000100:00000010:1.0:1788573891.925496:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d70c0.
00000020:00000001:1.0:1788573893.397547:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573893.397550:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573893.397553:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1788573896.086165:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573896.086167:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573896.086498:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573896.086500:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573896.086738:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573896.086740:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:1.0:1788573896.469557:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788573896.469560:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788573896.469564:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573897.045465:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305500601, at 4305500608
00000100:00000010:1.0:1788573897.045480:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7660.
00000800:00000200:0.1:1788573901.206129:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573901.206133:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573901.206622:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573901.206624:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788573901.206857:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573901.206859:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:1.0:1788573902.165485:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305505721, at 4305505728
00000100:00000010:1.0:1788573902.165503:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d80.
00000800:00000200:1.0:1788573906.325705:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788573906.325713:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573906.325725:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788573906.325731:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788573906.325735:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788573906.325932:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573906.325936:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788573906.326300:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573906.326302:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788573906.326547:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573906.326549:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:1.0:1788573907.285505:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305510841, at 4305510848
00000100:00000010:1.0:1788573907.285522:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000800:00000200:2.1:1788573911.446054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573911.446302:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573911.446492:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788573912.405487:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305515961, at 4305515968
00000100:00000010:1.0:1788573912.405503:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000400:00000100:1.0:1788573914.517653:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788573914.517698:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788573914.517707:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573914.517712:0:2350:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788573914.517717:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573914.517720:0:2350:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788573914.517726:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573914.517731:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:1.0:1788573914.517736:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573914.517741:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573914.517748:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65770
00000400:00000010:1.0:1788573914.517751:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65770.
00000100:00000001:1.0:1788573914.517756:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573914.517766:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348712680 x1875444393094912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573914.517794:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573914.517799:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393094912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573914.517807:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573914.517809:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573914.517818:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573914.517821:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133b00.
00000400:00000200:1.0:1788573914.517824:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573914.517829:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573914.517832:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:1.0:1788573914.517835:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:1.0:1788573914.517838:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573914.517843:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348710380 x1875444393095040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573914.517856:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573914.517862:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393095040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573914.517871:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573914.517872:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573914.517882:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573914.517888:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:1.0:1788573914.517894:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573914.517899:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573914.517902:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65550
00000400:00000010:1.0:1788573914.517908:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65550.
00000100:00000001:1.0:1788573914.517911:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573914.517919:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348712300 x1875444393095168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573914.517943:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573914.517951:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393095168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573914.517972:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573914.517976:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000200:2.1:1788573916.566109:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573916.566117:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573916.566671:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573916.566675:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788573916.567160:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573916.567164:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:1.0:1788573917.077548:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573917.077557:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573917.077560:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573917.077564:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573917.077568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573917.077571:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573917.077574:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573917.077576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573917.077578:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573917.525506:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305521081, at 4305521088
00000100:00000010:1.0:1788573917.525531:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000100:00000001:1.0:1788573918.101514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573918.101520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573918.101522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573918.101527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573918.101529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573918.101533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573918.101535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573918.101537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573918.101539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573919.125511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573919.125517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573919.125524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573919.125529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573919.125534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573919.125538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573919.125540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573919.125542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573919.125544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573920.149475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573920.149480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573920.149482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573920.149487:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573920.149489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573920.149492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573920.149495:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573920.149496:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573920.149499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573921.173509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573921.173516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573921.173518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573921.173530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573921.173532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573921.173538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573921.173540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573921.173542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573921.173545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573921.685527:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573921.685538:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573921.685541:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573921.685545:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573921.685551:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573921.685559:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10852
00000400:00000200:1.0:1788573921.685569:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573921.685576:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573921.685578:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573921.685581:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685583:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573921.685585:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685587:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573921.685589:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573921.685591:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573921.685593:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573921.685596:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573921.685598:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573921.685599:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573921.685601:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685602:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573921.685604:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685605:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573921.685607:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573921.685611:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685613:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573921.685617:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573921.685621:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573921.685624:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573921.685626:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573921.685629:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573921.685639:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788573921.685643:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573921.685648:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573921.685650:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573921.685653:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573921.685656:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573921.685659:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573921.685668:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788573921.685675:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573921.685684:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393095680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573921.685696:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573921.685701:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573921.685706:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393095680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573921.685728:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573921.685732:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573921.685736:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573921.685737:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573921.685739:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573921.685742:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573921.685746:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10852
00000400:00000200:1.0:1788573921.685752:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573921.685768:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573921.685774:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573921.685782:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685787:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573921.685793:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685798:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573921.685805:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573921.685810:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788573921.685815:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573921.685820:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573921.685826:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788573921.685832:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788573921.685839:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685845:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573921.685850:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685856:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573921.685862:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788573921.685870:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573921.685875:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573921.685880:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573921.685888:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573921.685893:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573921.685899:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573921.685904:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573921.685912:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573921.685916:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573921.685923:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788573921.685928:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573921.685934:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573921.685937:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573921.685939:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573921.685942:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573921.685945:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573921.685949:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788573921.685954:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573921.685962:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393095808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573921.685976:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573921.685979:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573921.685984:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393095808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573921.685995:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573921.686000:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573921.686006:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573921.686010:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573921.686013:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573921.686025:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573921.686031:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10852
00000400:00000200:1.0:1788573921.686040:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573921.686049:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573921.686054:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573921.686062:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.686068:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788573921.686072:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788573921.686075:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.1:1788573921.686076:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788573921.686082:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573921.686088:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573921.686092:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788573921.686095:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573921.686099:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788573921.686103:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788573921.686107:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573921.686115:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.686120:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573921.686128:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573921.686143:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573921.686150:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573921.686157:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573921.686161:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573921.686166:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573921.686171:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573921.686175:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573921.686179:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573921.686180:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573921.686183:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573921.686186:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573921.686190:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788573921.686194:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788573921.686196:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573921.686197:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573921.686198:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573921.686199:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573921.686200:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573921.686203:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788573921.686206:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573921.686215:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393095936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573921.686237:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573921.686240:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573921.686250:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393095936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573921.686259:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573921.686261:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788573921.686264:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788573921.686269:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10862)
00000800:00000200:2.1:1788573921.686539:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573921.686542:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788573921.686763:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788573921.686765:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788573922.197453:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573922.197460:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573922.197462:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573922.197492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573922.197495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573922.197498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573922.197501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573922.197503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573922.197505:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573922.645509:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305526201, at 4305526208
00000100:00000010:1.0:1788573922.645524:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000100:00000001:1.0:1788573923.221547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573923.221560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573923.221562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573923.221567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573923.221571:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573923.221575:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573923.221578:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573923.221581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573923.221584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573924.245513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573924.245519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573924.245521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573924.245525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573924.245529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573924.245532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573924.245534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573924.245535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573924.245537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573925.269465:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573925.269470:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573925.269471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573925.269475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573925.269478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573925.269481:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573925.269483:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573925.269486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573925.269488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573926.293498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573926.293504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573926.293506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573926.293534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573926.293538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573926.293542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573926.293545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573926.293548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573926.293558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573926.806003:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573926.806008:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1788573926.806355:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573926.806357:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1788573926.806675:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573926.806677:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:1.0:1788573927.317447:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573927.317453:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573927.317455:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573927.317460:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573927.317464:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573927.317467:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573927.317469:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573927.317471:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573927.317473:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573927.765462:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305531321, at 4305531328
00000100:00000010:1.0:1788573927.765477:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7840.
00000100:00000001:1.0:1788573928.341481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573928.341487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573928.341489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573928.341493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573928.341496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573928.341499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573928.341501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573928.341503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573928.341506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573929.365481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573929.365487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573929.365488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573929.365492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573929.365495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573929.365498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573929.365500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573929.365502:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573929.365504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573930.389579:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573930.389591:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573930.389594:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573930.389601:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573930.389607:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573930.389613:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573930.389616:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573930.389620:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573930.389624:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788573931.926074:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573931.926077:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788573931.926465:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573931.926466:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00080000:0.0:1788573931.926613:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573931.926616:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573931.926618:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573931.926620:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393096576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573956 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573931.926625:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393096576:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573931.926628:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573931.926630:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573931.926634:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573931.926635:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573931.926637:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573931.926638:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573931.926640:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573931.926641:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393096704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573931.926643:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573931.926645:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573931.926646:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573931.926647:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573931.926650:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788573931.926651:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573931.926654:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573931.926656:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573931.926659:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573931.926661:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393096704, portal 4
00000100:00000001:0.0:1788573931.926662:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573931.926663:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:0.0:1788573931.926667:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573931.926672:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573931.926674:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573931.926675:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573931.926677:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393096704, offset 0
00000400:00000200:0.0:1788573931.926679:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573931.926682:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573931.926685:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573931.926687:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573931.926689:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573931.926691:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573931.926692:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573931.926694:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330011:330011:256:4294967295] 192.168.202.110@tcp LPNI seq info [329968:329968:8:4294967295]
00000400:00000200:0.0:1788573931.926697:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573931.926702:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573931.926705:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573931.926707:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573931.926710:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573931.926712:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788573931.926715:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573931.926717:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1788573931.926719:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573931.926721:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573931.926723:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573931.926725:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573931.926728:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573931.926729:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573931.926730:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573931.926733:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573931.926738:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573931.926740:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573956 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573931.926744:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573931.926745:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573931.926748:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573931.926750:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573931.926752:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573931.926753:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305535673
00000100:00000001:0.0:1788573931.926755:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305536441
00000100:00000001:0.0:1788573931.926756:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305535673
00000100:00000001:0.0:1788573931.926758:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305536441
00000100:00000001:0.0:1788573931.926759:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573931.926760:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573931.926762:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573931.926764:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573931/real 1788573931]  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573931.926768:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573931.926769:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573931.926771:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573931.926772:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573931.926773:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573931.926774:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573931.926776:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573931.926781:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573931.926785:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573931.926786:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573931.926788:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573931.926789:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573931.926791:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573931.926795:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573931.926797:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573956 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573931.926802:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573931.926803:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573931.926804:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573931.926805:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573931.926806:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573931.926808:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573931.926809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573931.926811:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573931.926812:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573931.926815:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573956 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573931.926819:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393096704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573931.926821:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573931.926823:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573931.926825:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393096576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573956 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573931.926829:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573931.926830:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573931.926831:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573931.926832:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573931.926834:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573931.926835:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573931.926836:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573931.926837:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:0.0:1788573931.926839:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573931.926840:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573931.926841:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:0.0:1788573931.926843:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573931.926843:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573931.926845:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573931.926847:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393096704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573956 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573931.926851:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573931.926852:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573931.926853:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788573931.926854:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573931.926855:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573931.926856:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573931.926857:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573931.926857:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788573931.926859:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573931.926859:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573931.926860:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788573931.926862:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573931.926863:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573931.926864:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573931.926866:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573931.926867:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573931.926868:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573931.926870:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573932.309683:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573932.309687:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573932.309690:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573932.885493:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305536441, at 4305536448
00000100:00000040:0.0:1788573932.885494:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305536441, at 4305536448
00000100:00000010:1.0:1788573932.885508:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7600.
00000100:00000010:0.0:1788573932.885511:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7900.
00000020:00000001:0.0:1788573934.357559:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573934.357562:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573934.357565:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573935.381583:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573935.381587:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573935.381590:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788573937.045747:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573937.045753:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573937.045756:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573937.045765:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.045780:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573937.045783:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573937.045789:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573937.045794:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573937.045799:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573937.045808:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393096832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573937.045817:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573937.045824:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573937.045829:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573937.045834:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573937.045840:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573937.045843:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573937.045854:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573937.045858:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573937.045865:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573937.045873:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393096832, portal 25
00000100:00000001:0.0:1788573937.045876:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573937.045878:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:0.0:1788573937.045889:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.045897:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573937.045902:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573937.045905:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573937.045909:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393096832, offset 0
00000400:00000200:0.0:1788573937.045917:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573937.045927:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573937.045934:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573937.045942:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573937.045946:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573937.045951:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573937.045955:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573937.045961:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330012:330012:256:4294967295] 192.168.202.110@tcp LPNI seq info [329969:329969:8:4294967295]
00000400:00000200:0.0:1788573937.045968:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573937.045983:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573937.045992:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573937.045999:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573937.046008:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573937.046015:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788573937.046024:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573937.046027:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788573937.046034:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573937.046039:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573937.046046:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573937.046053:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573937.046059:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573937.046062:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573937.046068:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573937.046073:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046087:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573937.046093:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046108:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573937.046114:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573937.046124:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573937.046134:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573937.046140:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305541561
00000100:00000010:0.0:1788573937.046149:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7900.
00000100:00000001:0.0:1788573937.046159:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573937.046168:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305541561
00000100:00000010:0.0:1788573937.046175:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7f00.
00000100:00000001:0.0:1788573937.046181:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573937.046187:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573937.046191:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573937.046195:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573937.046204:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573937/real 1788573937]  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573937.046225:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573937.046229:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573937.046234:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573937.046239:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573937.046242:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573937.046245:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573937.046250:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573937.046276:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046300:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573937.046305:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.046309:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573937.046311:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573937.046319:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046329:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573937.046333:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046338:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573937.046340:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.046341:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573937.046342:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.046344:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573937.046345:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573937.046347:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573937.046349:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573937.046352:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573937.046354:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573937.046360:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393096832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573937.046363:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.046365:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573937.046367:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393096832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046371:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573937.046373:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573937.046375:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573937.046377:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573937.046378:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573937.046379:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573937.046380:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573937.046382:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788573937.046385:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573937.046386:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573937.046388:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:0.0:1788573937.046408:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573937.046409:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.046411:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.046417:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573937.046419:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573937.046420:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573937.046421:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573937.046422:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573937.046425:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046430:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573937.046431:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573937.046433:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573937.046434:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573937.046436:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573937.046438:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393096960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573937.046440:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573937.046442:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573937.046443:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573937.046444:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573937.046447:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573937.046448:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573937.046451:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573937.046453:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573937.046456:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573937.046458:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393096960, portal 10
00000100:00000001:0.0:1788573937.046460:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573937.046465:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:0.0:1788573937.046471:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046475:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573937.046476:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573937.046478:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573937.046480:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393096960, offset 0
00000400:00000200:0.0:1788573937.046482:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573937.046485:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573937.046488:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573937.046490:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573937.046492:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573937.046494:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573937.046496:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573937.046498:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330013:330013:256:4294967295] 192.168.202.110@tcp LPNI seq info [329970:329970:8:4294967295]
00000400:00000200:0.0:1788573937.046501:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573937.046505:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573937.046508:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573937.046511:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573937.046514:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573937.046516:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788573937.046519:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573937.046521:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788573937.046523:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573937.046525:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573937.046527:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573937.046529:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573937.046531:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573937.046533:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573937.046534:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573937.046537:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046542:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573937.046544:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046548:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573937.046550:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573937.046552:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573937.046554:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573937.046556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573937.046558:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305540647
00000100:00000001:0.0:1788573937.046560:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305541561
00000100:00000001:0.0:1788573937.046562:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305540647
00000100:00000001:0.0:1788573937.046563:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305541561
00000100:00000001:0.0:1788573937.046564:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573937.046566:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573937.046567:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573937.046569:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573937/real 1788573937]  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573937.046580:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573937.046582:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573937.046584:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573937.046586:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573937.046588:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573937.046590:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573937.046593:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573937.046602:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046609:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573937.046611:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.046613:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573937.046614:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573937.046618:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046626:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573937.046631:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046788:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573937.046791:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.046793:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573937.046795:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.046796:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573937.046798:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573937.046801:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573937.046806:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573937.046808:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573937.046813:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573937.046823:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393096960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573937.046827:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573937.046831:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046836:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573937.046838:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573937.046839:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573937.046841:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573937.046842:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573937.046844:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097088:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573937.046847:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573937.046848:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573937.046850:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573937.046851:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573937.046854:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788573937.046855:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573937.046858:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573937.046860:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573937.046862:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573937.046865:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393097088, portal 4
00000100:00000001:0.0:1788573937.046866:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573937.046867:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:0.0:1788573937.046871:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046875:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573937.046877:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573937.046879:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573937.046880:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393097088, offset 0
00000400:00000200:0.0:1788573937.046883:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573937.046886:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573937.046889:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573937.046891:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573937.046893:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573937.046895:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573937.046897:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573937.046899:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330014:330014:256:4294967295] 192.168.202.110@tcp LPNI seq info [329971:329971:8:4294967295]
00000400:00000200:0.0:1788573937.046902:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573937.046907:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573937.046910:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573937.046912:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573937.046914:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573937.046917:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788573937.046919:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573937.046921:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788573937.046924:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573937.046925:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573937.046928:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573937.046930:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573937.046933:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573937.046934:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573937.046936:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573937.046940:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046948:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573937.046951:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.046958:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573937.046960:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573937.046963:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573937.046966:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573937.046970:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573937.046972:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305540793
00000100:00000001:0.0:1788573937.046975:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305541561
00000100:00000001:0.0:1788573937.046978:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305540793
00000100:00000001:0.0:1788573937.046980:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305541561
00000100:00000001:0.0:1788573937.046982:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573937.046984:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573937.046987:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573937.046991:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573937/real 1788573937]  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573937.047000:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573937.047001:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573937.047003:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573937.047005:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573937.047007:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573937.047010:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573937.047013:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573937.047023:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.047030:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573937.047032:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.047034:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573937.047035:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573937.047039:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.047043:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573937.047046:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.047050:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573937.047053:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.047054:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573937.047055:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.047056:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573937.047057:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573937.047059:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573937.047061:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573937.047062:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573937.047065:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573937.047069:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097088:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573937.047072:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.047074:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573937.047076:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393096960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.047080:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573937.047081:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573937.047083:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573937.047084:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573937.047086:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573937.047087:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573937.047088:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573937.047089:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788573937.047091:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573937.047092:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573937.047093:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:0.0:1788573937.047095:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573937.047096:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.047097:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573937.047099:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393097088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573937.047107:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573937.047108:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573937.047109:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788573937.047110:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573937.047111:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573937.047112:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573937.047113:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573937.047113:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788573937.047115:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573937.047115:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573937.047116:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:0.0:1788573937.047117:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573937.047118:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.047119:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573937.047122:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573937.047123:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573937.047124:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573937.047125:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573937.429578:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573937.429582:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573937.429586:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573938.005449:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305541561, at 4305541568
00000100:00000010:1.0:1788573938.005462:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f00.
00000100:00000040:0.0:1788573938.005478:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305541561, at 4305541568
00000100:00000010:0.0:1788573938.005488:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7900.
00000020:00000001:0.0:1788573938.453569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573938.453575:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573938.453579:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573940.501570:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573940.501573:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573940.501576:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573941.525570:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573941.525574:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573941.525578:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788573942.165746:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573942.165752:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573942.165758:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573942.165770:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.165789:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573942.165794:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573942.165799:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573942.165805:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573942.165809:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573942.165815:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393097216:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573942.165823:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573942.165829:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573942.165833:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573942.165838:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573942.165845:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573942.165849:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573942.165860:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573942.165866:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573942.165874:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573942.165884:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393097216, portal 25
00000100:00000001:0.0:1788573942.165889:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573942.165894:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:0.0:1788573942.165907:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.165928:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573942.165936:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573942.165948:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573942.165956:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393097216, offset 0
00000400:00000200:0.0:1788573942.165970:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573942.165982:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573942.165993:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573942.166001:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573942.166008:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573942.166014:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573942.166020:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573942.166029:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330015:330015:256:4294967295] 192.168.202.110@tcp LPNI seq info [329972:329972:8:4294967295]
00000400:00000200:0.0:1788573942.166040:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573942.166061:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573942.166073:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573942.166083:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573942.166096:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573942.166104:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573942.166111:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573942.166115:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1788573942.166126:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573942.166129:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573942.166136:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573942.166141:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573942.166147:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573942.166150:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573942.166154:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573942.166159:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166170:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573942.166174:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166182:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573942.166184:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573942.166190:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573942.166195:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573942.166198:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305546681
00000100:00000010:0.0:1788573942.166204:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573942.166211:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573942.166217:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305546681
00000100:00000010:0.0:1788573942.166221:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7a80.
00000100:00000001:0.0:1788573942.166224:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573942.166228:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573942.166231:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573942.166234:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573942.166239:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573942/real 1788573942]  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573942.166250:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573942.166252:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573942.166255:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573942.166258:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573942.166260:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573942.166263:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573942.166267:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573942.166277:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166285:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573942.166286:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.166289:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573942.166291:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573942.166296:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166304:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573942.166309:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166317:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573942.166321:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.166323:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573942.166324:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.166327:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573942.166329:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573942.166332:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573942.166336:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573942.166340:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573942.166345:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573942.166353:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393097216:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573942.166358:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.166361:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573942.166364:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393097216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166372:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573942.166375:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573942.166378:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573942.166381:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573942.166383:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573942.166385:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573942.166388:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573942.166410:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788573942.166414:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573942.166417:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573942.166419:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:0.0:1788573942.166422:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573942.166424:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.166426:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.166434:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573942.166436:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573942.166439:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573942.166441:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573942.166442:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573942.166447:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166455:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573942.166457:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573942.166460:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573942.166462:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573942.166464:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573942.166468:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097344:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573942.166472:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573942.166475:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573942.166477:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573942.166479:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573942.166482:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573942.166484:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573942.166488:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573942.166491:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573942.166496:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573942.166499:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393097344, portal 10
00000100:00000001:0.0:1788573942.166501:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573942.166503:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:0.0:1788573942.166510:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166518:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573942.166521:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573942.166523:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573942.166526:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393097344, offset 0
00000400:00000200:0.0:1788573942.166531:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573942.166535:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573942.166539:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573942.166543:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573942.166546:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573942.166548:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573942.166551:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573942.166555:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330016:330016:256:4294967295] 192.168.202.110@tcp LPNI seq info [329973:329973:8:4294967295]
00000400:00000200:0.0:1788573942.166560:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573942.166567:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573942.166572:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573942.166576:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573942.166580:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573942.166583:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573942.166588:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573942.166591:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1788573942.166596:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573942.166598:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573942.166601:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573942.166605:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573942.166608:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573942.166610:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573942.166613:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573942.166618:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166627:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573942.166630:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166638:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573942.166640:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573942.166643:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573942.166646:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573942.166650:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573942.166652:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305545767
00000100:00000001:0.0:1788573942.166655:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305546681
00000100:00000001:0.0:1788573942.166658:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305545767
00000100:00000001:0.0:1788573942.166660:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305546681
00000100:00000001:0.0:1788573942.166662:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573942.166665:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573942.166667:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573942.166671:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573942/real 1788573942]  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573942.166687:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573942.166689:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573942.166691:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573942.166707:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573942.166710:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573942.166712:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573942.166716:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573942.166726:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166735:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573942.166736:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.166739:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573942.166740:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573942.166744:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166752:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573942.166756:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166765:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573942.166767:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.166769:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573942.166771:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.166773:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573942.166775:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573942.166778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573942.166782:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573942.166785:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573942.166790:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573942.166799:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097344:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573942.166803:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573942.166807:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166815:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573942.166817:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573942.166819:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573942.166821:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573942.166823:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573942.166827:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097472:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573942.166830:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573942.166832:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573942.166834:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573942.166836:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573942.166840:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788573942.166843:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573942.166846:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573942.166849:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573942.166853:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573942.166856:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393097472, portal 4
00000100:00000001:0.0:1788573942.166858:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573942.166860:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:0.0:1788573942.166866:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166874:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573942.166877:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573942.166879:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573942.166882:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393097472, offset 0
00000400:00000200:0.0:1788573942.166887:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573942.166891:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573942.166896:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573942.166899:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573942.166901:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573942.166904:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573942.166907:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573942.166910:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330017:330017:256:4294967295] 192.168.202.110@tcp LPNI seq info [329974:329974:8:4294967295]
00000400:00000200:0.0:1788573942.166915:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573942.166922:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573942.166928:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573942.166931:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573942.166935:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573942.166938:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788573942.166943:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573942.166946:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1788573942.166950:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573942.166952:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573942.166956:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573942.166959:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573942.166963:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573942.166965:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573942.166968:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573942.166972:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166980:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573942.166984:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.166991:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573942.166993:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573942.166996:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573942.166999:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573942.167003:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573942.167006:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305545913
00000100:00000001:0.0:1788573942.167009:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305546681
00000100:00000001:0.0:1788573942.167011:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305545913
00000100:00000001:0.0:1788573942.167014:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305546681
00000100:00000001:0.0:1788573942.167016:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573942.167018:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573942.167020:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573942.167024:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573942/real 1788573942]  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573942.167033:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573942.167034:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573942.167036:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573942.167038:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573942.167041:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573942.167043:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573942.167047:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573942.167057:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.167064:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573942.167066:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.167068:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573942.167070:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573942.167074:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.167082:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573942.167087:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.167095:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573942.167098:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.167100:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573942.167101:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.167103:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573942.167105:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573942.167108:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573942.167112:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573942.167114:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573942.167118:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573942.167127:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097472:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573942.167131:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.167134:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573942.167137:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393097344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.167145:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573942.167147:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573942.167149:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573942.167152:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573942.167154:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573942.167155:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573942.167158:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573942.167160:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788573942.167163:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573942.167165:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573942.167166:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:0.0:1788573942.167169:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573942.167171:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.167173:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573942.167177:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393097472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573942.167190:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573942.167191:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573942.167193:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788573942.167196:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573942.167197:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573942.167199:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573942.167201:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573942.167203:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788573942.167205:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573942.167206:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573942.167208:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:0.0:1788573942.167210:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573942.167212:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.167214:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573942.167217:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573942.167219:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573942.167221:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573942.167223:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573943.125477:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305546681, at 4305546688
00000100:00000040:0.0:1788573943.125478:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305546681, at 4305546688
00000100:00000010:0.0:1788573943.125494:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788573943.125497:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7a80.
00000020:00000001:0.0:1788573943.573625:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573943.573629:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573943.573633:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573944.597579:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573944.597582:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573944.597585:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.0:1788573945.237509:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788573945.237519:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788573945.237525:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788573945.237671:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788573945.237686:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:1.0:1788573945.237686:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788573945.237694:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788573945.237698:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1788573946.645605:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573946.645609:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573946.645613:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788573947.285795:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573947.285803:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573947.285809:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573947.285821:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.285847:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573947.285852:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573947.285862:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573947.285867:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573947.285874:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573947.285886:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393097600:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573947.285898:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573947.285907:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573947.285911:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573947.285916:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573947.285924:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573947.285929:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573947.285943:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573947.285950:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573947.285961:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573947.285970:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393097600, portal 25
00000100:00000001:0.0:1788573947.285976:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573947.285979:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:0.0:1788573947.285991:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286010:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573947.286018:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573947.286026:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573947.286034:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393097600, offset 0
00000400:00000200:0.0:1788573947.286051:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573947.286069:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573947.286084:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573947.286095:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573947.286101:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573947.286116:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573947.286120:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573947.286125:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330018:330018:256:4294967295] 192.168.202.110@tcp LPNI seq info [329975:329975:8:4294967295]
00000400:00000200:0.0:1788573947.286131:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573947.286143:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573947.286152:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573947.286159:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573947.286167:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573947.286170:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573947.286176:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573947.286179:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1788573947.286182:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573947.286185:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573947.286193:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573947.286197:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573947.286203:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573947.286205:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573947.286209:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573947.286212:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286220:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573947.286223:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573972 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286229:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573947.286230:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573947.286236:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573947.286239:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573947.286242:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305551801
00000100:00000010:0.0:1788573947.286247:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573947.286255:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573947.286261:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305551801
00000100:00000010:0.0:1788573947.286263:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d78a0.
00000100:00000001:0.0:1788573947.286266:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573947.286269:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573947.286271:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573947.286273:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573947.286276:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573947/real 1788573947]  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573947.286285:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573947.286287:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573947.286289:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573947.286291:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573947.286292:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573947.286294:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573947.286296:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573947.286302:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286306:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573947.286307:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286309:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573947.286311:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573947.286314:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286318:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573947.286322:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286327:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573947.286330:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286339:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573947.286341:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286343:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573947.286345:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573947.286348:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573947.286351:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573947.286353:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573947.286356:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573947.286362:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393097600:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573947.286366:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286367:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573947.286369:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393097600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286373:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573947.286376:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573947.286378:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573947.286380:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573947.286382:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573947.286383:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573947.286385:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573947.286388:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:0.0:1788573947.286409:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573947.286411:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573947.286413:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:0.0:1788573947.286416:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573947.286417:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286419:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286426:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573947.286428:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573947.286429:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573947.286431:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573947.286432:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573947.286436:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286445:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573947.286446:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573947.286449:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573947.286451:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573947.286453:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573947.286457:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573947.286461:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573947.286463:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573947.286465:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573947.286467:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573947.286470:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573947.286472:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573947.286477:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573947.286480:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573947.286483:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573947.286486:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393097728, portal 10
00000100:00000001:0.0:1788573947.286487:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573947.286488:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:0.0:1788573947.286495:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286499:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573947.286501:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573947.286503:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573947.286504:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393097728, offset 0
00000400:00000200:0.0:1788573947.286507:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573947.286510:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573947.286513:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573947.286515:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573947.286517:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573947.286518:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573947.286520:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573947.286523:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330019:330019:256:4294967295] 192.168.202.110@tcp LPNI seq info [329976:329976:8:4294967295]
00000400:00000200:0.0:1788573947.286526:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573947.286531:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573947.286534:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573947.286537:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573947.286539:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573947.286542:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573947.286545:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573947.286547:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1788573947.286550:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573947.286552:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573947.286554:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573947.286556:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573947.286559:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573947.286560:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573947.286562:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573947.286564:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286570:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573947.286572:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573972 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286577:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573947.286578:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573947.286580:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573947.286583:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573947.286585:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573947.286587:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305550887
00000100:00000001:0.0:1788573947.286589:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305551801
00000100:00000001:0.0:1788573947.286598:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305550887
00000100:00000001:0.0:1788573947.286600:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305551801
00000100:00000001:0.0:1788573947.286602:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573947.286604:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573947.286606:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573947.286609:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573947/real 1788573947]  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573947.286615:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573947.286616:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573947.286617:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573947.286619:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573947.286620:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573947.286621:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573947.286623:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573947.286629:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286633:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573947.286634:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286635:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573947.286636:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573947.286638:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286642:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573947.286644:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286649:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573947.286651:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286652:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573947.286653:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286654:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573947.286656:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573947.286657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573947.286660:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573947.286661:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573947.286665:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573947.286670:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573947.286672:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573947.286680:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286684:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573947.286685:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573947.286686:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573947.286687:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573947.286689:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573947.286691:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573947.286692:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573947.286694:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573947.286695:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573947.286696:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573947.286698:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788573947.286700:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573947.286702:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573947.286704:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573947.286706:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573947.286708:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393097856, portal 4
00000100:00000001:0.0:1788573947.286710:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573947.286711:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:0.0:1788573947.286715:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286721:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573947.286724:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573947.286726:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573947.286728:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393097856, offset 0
00000400:00000200:0.0:1788573947.286733:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573947.286737:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573947.286741:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573947.286744:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573947.286747:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573947.286749:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573947.286752:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573947.286767:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330020:330020:256:4294967295] 192.168.202.110@tcp LPNI seq info [329977:329977:8:4294967295]
00000400:00000200:0.0:1788573947.286772:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573947.286779:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573947.286785:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573947.286790:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573947.286794:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573947.286798:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788573947.286803:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573947.286806:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1788573947.286811:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573947.286813:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573947.286817:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573947.286821:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573947.286825:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573947.286827:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573947.286830:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573947.286834:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286843:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573947.286847:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573972 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286855:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573947.286858:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573947.286861:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573947.286865:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573947.286869:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573947.286871:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305551033
00000100:00000001:0.0:1788573947.286874:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305551801
00000100:00000001:0.0:1788573947.286876:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305551033
00000100:00000001:0.0:1788573947.286879:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305551801
00000100:00000001:0.0:1788573947.286881:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573947.286884:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573947.286886:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573947.286890:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573947/real 1788573947]  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573947.286899:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573947.286901:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573947.286903:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573947.286906:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573947.286908:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573947.286910:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573947.286914:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573947.286921:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286926:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573947.286927:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286928:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573947.286929:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573947.286932:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286936:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573947.286939:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286944:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573947.286946:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286947:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573947.286948:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286949:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573947.286951:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573947.286952:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573947.286954:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573947.286956:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573947.286958:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573947.286963:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393097856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573947.286966:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286968:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573947.286970:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393097728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286974:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573947.286976:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573947.286977:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573947.286979:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573947.286980:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573947.286981:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573947.286983:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573947.286984:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:0.0:1788573947.286986:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573947.286987:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573947.286988:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:0.0:1788573947.286990:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573947.286991:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.286993:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573947.286995:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393097856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573972 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573947.286999:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573947.287000:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573947.287001:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788573947.287002:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573947.287003:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573947.287004:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573947.287005:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573947.287006:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788573947.287007:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573947.287007:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573947.287008:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:0.0:1788573947.287010:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573947.287010:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.287012:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573947.287014:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573947.287015:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573947.287016:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573947.287017:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573947.669566:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573947.669571:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573947.669575:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573948.245492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305551801, at 4305551808
00000100:00000040:0.0:1788573948.245499:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305551801, at 4305551808
00000100:00000010:1.0:1788573948.245506:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d78a0.
00000100:00000010:0.0:1788573948.245509:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788573949.717689:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573949.717695:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573949.717699:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573950.741541:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573950.741546:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573950.741549:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788573952.405854:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573952.405861:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573952.405864:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573952.405874:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.405887:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573952.405891:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573952.405896:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573952.405900:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573952.405903:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573952.405909:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393097984:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573952.405914:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573952.405920:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573952.405923:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573952.405926:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573952.405934:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573952.405938:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573952.405948:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573952.405954:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573952.405962:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573952.405971:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393097984, portal 25
00000100:00000001:0.0:1788573952.405975:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573952.405978:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:0.0:1788573952.405986:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.405996:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573952.406001:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573952.406006:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573952.406011:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393097984, offset 0
00000400:00000200:0.0:1788573952.406021:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573952.406034:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573952.406048:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573952.406060:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573952.406070:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573952.406079:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573952.406100:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573952.406108:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330021:330021:256:4294967295] 192.168.202.110@tcp LPNI seq info [329978:329978:8:4294967295]
00000400:00000200:0.0:1788573952.406118:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573952.406138:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573952.406150:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573952.406158:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573952.406168:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573952.406173:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573952.406180:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573952.406183:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788573952.406187:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573952.406190:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573952.406197:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573952.406203:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573952.406209:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573952.406211:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573952.406216:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573952.406221:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406229:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573952.406232:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406237:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573952.406238:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573952.406244:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573952.406247:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573952.406250:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305556921
00000100:00000010:0.0:1788573952.406255:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573952.406262:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573952.406268:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305556921
00000100:00000010:0.0:1788573952.406271:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788573952.406274:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573952.406282:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573952.406285:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573952.406287:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573952.406290:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573952/real 1788573952]  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573952.406299:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573952.406301:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573952.406304:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573952.406306:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573952.406307:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573952.406308:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573952.406310:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573952.406316:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406321:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573952.406322:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406324:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573952.406326:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573952.406329:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406333:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573952.406336:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406341:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573952.406343:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406344:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573952.406345:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406346:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573952.406348:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573952.406350:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573952.406353:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573952.406356:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573952.406359:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573952.406365:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393097984:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573952.406369:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406370:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573952.406372:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393097984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406376:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573952.406378:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573952.406380:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573952.406382:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573952.406384:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573952.406385:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573952.406386:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573952.406389:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788573952.406414:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573952.406415:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573952.406417:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:0.0:1788573952.406420:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573952.406421:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406422:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406429:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573952.406431:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573952.406433:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573952.406434:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573952.406435:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573952.406438:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406443:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573952.406444:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573952.406446:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573952.406448:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573952.406450:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573952.406452:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098112:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573952.406454:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573952.406456:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573952.406457:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573952.406458:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573952.406461:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573952.406462:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573952.406464:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573952.406467:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573952.406469:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573952.406471:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393098112, portal 10
00000100:00000001:0.0:1788573952.406473:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573952.406474:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:0.0:1788573952.406479:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406484:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573952.406485:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573952.406487:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573952.406488:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393098112, offset 0
00000400:00000200:0.0:1788573952.406491:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573952.406493:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573952.406496:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573952.406498:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573952.406499:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573952.406501:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573952.406503:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573952.406505:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330022:330022:256:4294967295] 192.168.202.110@tcp LPNI seq info [329979:329979:8:4294967295]
00000400:00000200:0.0:1788573952.406508:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573952.406512:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573952.406515:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573952.406518:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573952.406520:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573952.406522:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573952.406525:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573952.406527:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788573952.406531:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573952.406532:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573952.406534:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573952.406537:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573952.406539:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573952.406540:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573952.406542:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573952.406544:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406549:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573952.406551:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406556:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573952.406557:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573952.406559:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573952.406561:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573952.406563:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573952.406565:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305556007
00000100:00000001:0.0:1788573952.406567:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305556921
00000100:00000001:0.0:1788573952.406568:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305556007
00000100:00000001:0.0:1788573952.406570:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305556921
00000100:00000001:0.0:1788573952.406572:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573952.406573:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573952.406574:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573952.406576:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573952/real 1788573952]  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573952.406586:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573952.406587:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573952.406588:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573952.406590:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573952.406591:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573952.406592:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573952.406594:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573952.406599:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406603:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573952.406604:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406605:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573952.406606:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573952.406608:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406613:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573952.406615:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406619:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573952.406620:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406621:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573952.406622:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406623:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573952.406624:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573952.406626:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573952.406628:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573952.406629:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573952.406633:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573952.406638:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098112:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573952.406641:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573952.406643:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406647:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573952.406647:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573952.406649:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573952.406650:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573952.406651:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573952.406653:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098240:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573952.406654:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573952.406655:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573952.406656:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573952.406657:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573952.406660:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788573952.406661:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573952.406663:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573952.406665:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573952.406668:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573952.406669:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393098240, portal 4
00000100:00000001:0.0:1788573952.406671:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573952.406672:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:0.0:1788573952.406676:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406680:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573952.406682:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573952.406683:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573952.406685:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393098240, offset 0
00000400:00000200:0.0:1788573952.406688:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573952.406690:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573952.406692:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573952.406694:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573952.406696:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573952.406697:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573952.406699:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573952.406701:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330023:330023:256:4294967295] 192.168.202.110@tcp LPNI seq info [329980:329980:8:4294967295]
00000400:00000200:0.0:1788573952.406704:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573952.406708:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573952.406710:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573952.406712:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573952.406714:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573952.406716:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788573952.406719:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573952.406721:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788573952.406723:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573952.406725:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573952.406727:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573952.406729:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573952.406731:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573952.406733:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573952.406734:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573952.406736:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406741:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573952.406743:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406747:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573952.406748:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573952.406749:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573952.406751:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573952.406753:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573952.406755:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305556153
00000100:00000001:0.0:1788573952.406757:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305556921
00000100:00000001:0.0:1788573952.406759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305556153
00000100:00000001:0.0:1788573952.406760:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305556921
00000100:00000001:0.0:1788573952.406761:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573952.406763:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573952.406764:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573952.406766:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573952/real 1788573952]  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573952.406770:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573952.406771:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573952.406772:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573952.406773:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573952.406774:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573952.406776:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573952.406778:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573952.406784:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406788:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573952.406788:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406789:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573952.406790:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573952.406792:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406796:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573952.406798:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406802:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573952.406804:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406806:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573952.406806:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406808:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573952.406809:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573952.406811:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573952.406829:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573952.406831:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573952.406834:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573952.406839:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098240:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573952.406842:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406843:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573952.406845:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393098112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406849:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573952.406850:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573952.406852:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573952.406854:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573952.406855:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573952.406856:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573952.406857:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573952.406859:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788573952.406861:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573952.406861:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573952.406863:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:0.0:1788573952.406865:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573952.406866:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406867:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573952.406869:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393098240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573952.406877:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573952.406878:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573952.406879:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788573952.406881:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573952.406882:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573952.406882:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573952.406883:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573952.406884:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:0.0:1788573952.406885:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573952.406886:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573952.406887:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:0.0:1788573952.406888:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573952.406889:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406890:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573952.406892:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573952.406893:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573952.406894:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573952.406895:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573952.789637:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573952.789641:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573952.789645:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788573953.365487:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305556921, at 4305556928
00000100:00000040:1.0:1788573953.365487:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305556921, at 4305556928
00000100:00000010:1.0:1788573953.365506:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:0.0:1788573953.365507:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788573953.813631:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573953.813635:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573953.813638:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573955.861615:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573955.861621:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573955.861625:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573956.885594:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573956.885597:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573956.885600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788573957.525754:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573957.525761:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573957.525767:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573957.525781:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.525800:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573957.525805:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573957.525811:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573957.525817:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573957.525823:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573957.525832:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393098368:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573957.525840:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573957.525846:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573957.525850:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573957.525853:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573957.525860:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573957.525865:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573957.525889:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573957.525892:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573957.525899:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573957.525906:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393098368, portal 25
00000100:00000001:0.0:1788573957.525912:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573957.525915:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:0.0:1788573957.525924:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.525938:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573957.525945:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573957.525949:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573957.525954:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393098368, offset 0
00000400:00000200:0.0:1788573957.525963:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573957.525975:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573957.525983:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573957.525993:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573957.525999:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573957.526006:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573957.526016:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573957.526027:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330024:330024:256:4294967295] 192.168.202.110@tcp LPNI seq info [329981:329981:8:4294967295]
00000400:00000200:0.0:1788573957.526043:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573957.526063:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573957.526074:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573957.526081:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573957.526092:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573957.526097:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573957.526107:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573957.526112:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788573957.526123:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573957.526126:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573957.526132:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573957.526137:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573957.526141:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573957.526143:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573957.526147:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573957.526151:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526158:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573957.526161:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526166:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573957.526167:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573957.526172:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573957.526175:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573957.526177:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305562041
00000100:00000010:0.0:1788573957.526182:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573957.526187:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573957.526192:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305562041
00000100:00000010:0.0:1788573957.526194:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7c00.
00000100:00000001:0.0:1788573957.526196:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573957.526199:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573957.526201:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573957.526203:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573957.526206:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573957/real 1788573957]  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573957.526213:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573957.526214:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573957.526216:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573957.526218:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573957.526219:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573957.526220:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573957.526223:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573957.526229:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526233:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573957.526234:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526236:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573957.526237:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573957.526240:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526244:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573957.526247:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526252:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573957.526254:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526255:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573957.526255:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526257:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573957.526258:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573957.526260:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573957.526263:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573957.526265:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573957.526268:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573957.526274:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393098368:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573957.526277:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526279:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573957.526280:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393098368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526293:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573957.526295:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573957.526297:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573957.526298:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573957.526300:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573957.526301:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573957.526302:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573957.526304:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:0.0:1788573957.526306:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573957.526307:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573957.526309:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:0.0:1788573957.526311:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573957.526312:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526313:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526319:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573957.526321:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573957.526322:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573957.526323:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573957.526324:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573957.526327:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526332:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573957.526333:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573957.526334:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573957.526336:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573957.526337:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573957.526339:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098496:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573957.526341:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573957.526343:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573957.526344:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573957.526345:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573957.526347:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573957.526348:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573957.526350:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573957.526352:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573957.526355:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573957.526357:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393098496, portal 10
00000100:00000001:0.0:1788573957.526358:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573957.526359:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:0.0:1788573957.526363:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526369:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573957.526371:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573957.526373:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573957.526376:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393098496, offset 0
00000400:00000200:0.0:1788573957.526380:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573957.526384:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573957.526388:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573957.526414:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573957.526417:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573957.526420:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573957.526423:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573957.526426:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330025:330025:256:4294967295] 192.168.202.110@tcp LPNI seq info [329982:329982:8:4294967295]
00000400:00000200:0.0:1788573957.526429:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573957.526433:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573957.526437:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573957.526440:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573957.526443:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573957.526445:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573957.526448:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573957.526450:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788573957.526452:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573957.526454:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573957.526456:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573957.526458:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573957.526461:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573957.526462:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573957.526464:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573957.526466:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526472:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573957.526474:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526479:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573957.526480:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573957.526482:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573957.526485:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573957.526487:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573957.526489:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305561127
00000100:00000001:0.0:1788573957.526491:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305562041
00000100:00000001:0.0:1788573957.526492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305561127
00000100:00000001:0.0:1788573957.526494:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305562041
00000100:00000001:0.0:1788573957.526495:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573957.526497:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573957.526498:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573957.526500:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573957/real 1788573957]  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573957.526510:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573957.526511:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573957.526513:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573957.526514:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573957.526515:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573957.526516:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573957.526519:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573957.526524:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526528:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573957.526529:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526530:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573957.526531:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573957.526533:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526538:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573957.526542:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526549:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573957.526552:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526554:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573957.526555:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526557:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573957.526559:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573957.526562:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573957.526566:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573957.526569:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573957.526574:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573957.526582:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098496:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573957.526586:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573957.526590:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526598:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573957.526599:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573957.526602:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573957.526603:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573957.526606:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573957.526608:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573957.526611:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573957.526614:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573957.526615:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573957.526616:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573957.526619:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788573957.526620:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573957.526623:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573957.526625:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573957.526627:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573957.526630:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393098624, portal 4
00000100:00000001:0.0:1788573957.526631:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573957.526633:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:0.0:1788573957.526639:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526646:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573957.526648:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573957.526651:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573957.526654:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393098624, offset 0
00000400:00000200:0.0:1788573957.526658:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573957.526663:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573957.526666:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573957.526669:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573957.526672:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573957.526675:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573957.526677:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573957.526681:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330026:330026:256:4294967295] 192.168.202.110@tcp LPNI seq info [329983:329983:8:4294967295]
00000400:00000200:0.0:1788573957.526694:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573957.526701:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573957.526706:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573957.526709:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573957.526713:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573957.526716:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788573957.526720:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573957.526723:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788573957.526727:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573957.526729:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573957.526732:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573957.526736:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573957.526739:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573957.526741:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573957.526744:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573957.526748:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526757:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573957.526760:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526767:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573957.526769:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573957.526772:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573957.526775:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573957.526779:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573957.526781:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305561273
00000100:00000001:0.0:1788573957.526784:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305562041
00000100:00000001:0.0:1788573957.526786:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305561273
00000100:00000001:0.0:1788573957.526788:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305562041
00000100:00000001:0.0:1788573957.526791:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573957.526793:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573957.526795:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573957.526799:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573957/real 1788573957]  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573957.526807:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573957.526809:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573957.526811:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573957.526814:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573957.526815:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573957.526818:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573957.526821:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573957.526830:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526837:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573957.526838:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526841:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573957.526842:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573957.526847:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526855:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573957.526860:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526867:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573957.526885:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526887:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573957.526889:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526891:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573957.526893:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573957.526896:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573957.526900:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573957.526902:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573957.526906:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573957.526915:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573957.526919:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526921:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573957.526925:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393098496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526932:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573957.526934:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573957.526936:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573957.526939:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573957.526941:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573957.526943:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573957.526945:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573957.526947:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:0.0:1788573957.526950:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573957.526951:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573957.526952:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:0.0:1788573957.526955:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573957.526957:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526958:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573957.526962:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393098624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573957.526975:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573957.526977:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573957.526979:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788573957.526982:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573957.526983:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573957.526984:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573957.526986:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573957.526988:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:0.0:1788573957.526990:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573957.526991:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573957.526993:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:0.0:1788573957.526995:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573957.526996:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.526998:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573957.527002:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573957.527003:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573957.527005:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573957.527007:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573958.485488:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305562041, at 4305562048
00000100:00000040:1.0:1788573958.485488:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305562041, at 4305562048
00000100:00000010:0.0:1788573958.485503:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788573958.485506:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7c00.
00000020:00000001:0.0:1788573958.933609:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573958.933613:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573958.933617:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573959.957579:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573959.957582:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573959.957585:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573960.085492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573960.085498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573960.085499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573960.085504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573960.085507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573960.085510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573960.085512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573960.085516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573960.085520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573961.109459:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573961.109467:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573961.109469:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573961.109475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573961.109478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573961.109482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573961.109485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573961.109488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573961.109491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573962.005667:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573962.005674:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573962.005679:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573962.133484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573962.133491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573962.133494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573962.133499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573962.133503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573962.133506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573962.133509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573962.133512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573962.133516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573962.645841:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573962.645847:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573962.645851:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573962.645861:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.645879:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573962.645885:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573962.645894:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573962.645898:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573962.645905:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573962.645916:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393098752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573962.645924:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573962.645942:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573962.645945:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573962.645949:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573962.645958:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573962.645962:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573962.645972:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573962.645978:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573962.645989:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573962.645998:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393098752, portal 25
00000100:00000001:0.0:1788573962.646010:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573962.646013:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000040:0.0:1788573962.646025:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646037:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573962.646044:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573962.646049:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573962.646053:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393098752, offset 0
00000400:00000200:0.0:1788573962.646063:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573962.646076:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573962.646083:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573962.646091:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573962.646098:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573962.646105:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573962.646112:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573962.646122:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330027:330027:256:4294967295] 192.168.202.110@tcp LPNI seq info [329984:329984:8:4294967295]
00000400:00000200:0.0:1788573962.646135:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573962.646163:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573962.646177:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573962.646185:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573962.646196:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573962.646201:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573962.646212:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573962.646218:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788573962.646227:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573962.646230:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573962.646236:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573962.646247:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573962.646259:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573962.646265:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573962.646275:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573962.646286:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646300:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573962.646304:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646310:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573962.646312:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573962.646319:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573962.646330:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573962.646336:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305567161
00000100:00000010:0.0:1788573962.646340:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573962.646347:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573962.646353:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305567161
00000100:00000010:0.0:1788573962.646355:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7cc0.
00000100:00000001:0.0:1788573962.646358:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573962.646360:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573962.646363:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573962.646365:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573962.646369:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573962/real 1788573962]  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573962.646382:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573962.646384:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573962.646386:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573962.646388:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573962.646389:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573962.646410:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573962.646413:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573962.646420:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646425:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573962.646426:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646428:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573962.646431:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573962.646435:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646443:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573962.646449:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646458:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573962.646461:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646463:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573962.646465:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646468:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573962.646470:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573962.646474:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573962.646478:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573962.646481:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573962.646485:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573962.646491:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393098752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573962.646496:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646498:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573962.646499:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393098752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646504:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573962.646507:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573962.646509:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573962.646512:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573962.646513:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573962.646515:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573962.646516:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573962.646519:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:0.0:1788573962.646522:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573962.646524:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573962.646526:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:0.0:1788573962.646528:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573962.646529:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646531:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646539:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573962.646542:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573962.646545:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573962.646546:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573962.646548:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573962.646552:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646560:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573962.646561:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573962.646564:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573962.646566:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573962.646568:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573962.646572:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573962.646577:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573962.646579:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573962.646581:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573962.646583:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573962.646586:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573962.646589:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573962.646592:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573962.646594:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573962.646598:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573962.646600:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393098880, portal 10
00000100:00000001:0.0:1788573962.646602:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573962.646603:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:0.0:1788573962.646609:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646614:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573962.646615:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573962.646617:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573962.646618:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393098880, offset 0
00000400:00000200:0.0:1788573962.646621:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573962.646624:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573962.646626:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573962.646628:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573962.646630:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573962.646632:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573962.646634:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573962.646636:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330028:330028:256:4294967295] 192.168.202.110@tcp LPNI seq info [329985:329985:8:4294967295]
00000400:00000200:0.0:1788573962.646639:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573962.646644:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573962.646647:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573962.646649:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573962.646652:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573962.646655:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573962.646658:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573962.646660:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788573962.646662:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573962.646664:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573962.646666:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573962.646669:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573962.646672:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573962.646673:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573962.646675:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573962.646677:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646683:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573962.646685:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646690:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573962.646691:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573962.646693:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573962.646696:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573962.646698:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573962.646700:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305566247
00000100:00000001:0.0:1788573962.646702:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305567161
00000100:00000001:0.0:1788573962.646704:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305566247
00000100:00000001:0.0:1788573962.646706:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305567161
00000100:00000001:0.0:1788573962.646707:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573962.646708:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573962.646709:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573962.646712:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573962/real 1788573962]  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573962.646717:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573962.646718:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573962.646719:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573962.646721:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573962.646722:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573962.646724:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573962.646726:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573962.646731:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646735:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573962.646736:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646737:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573962.646738:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573962.646740:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646744:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573962.646747:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646751:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573962.646753:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646754:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573962.646755:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646756:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573962.646758:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573962.646760:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573962.646762:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573962.646764:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573962.646766:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573962.646771:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393098880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573962.646774:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573962.646776:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646781:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573962.646782:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573962.646783:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573962.646785:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573962.646786:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573962.646787:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099008:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573962.646789:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573962.646791:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573962.646792:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573962.646793:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573962.646795:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573962.646797:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573962.646799:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573962.646801:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573962.646803:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573962.646805:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393099008, portal 4
00000100:00000001:0.0:1788573962.646807:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573962.646808:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:0.0:1788573962.646812:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646817:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573962.646818:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573962.646820:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573962.646822:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393099008, offset 0
00000400:00000200:0.0:1788573962.646824:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573962.646826:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573962.646828:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573962.646830:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573962.646832:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573962.646834:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573962.646835:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573962.646837:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330029:330029:256:4294967295] 192.168.202.110@tcp LPNI seq info [329986:329986:8:4294967295]
00000400:00000200:0.0:1788573962.646840:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573962.646845:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573962.646847:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573962.646849:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573962.646851:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573962.646854:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788573962.646856:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573962.646858:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788573962.646861:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573962.646862:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573962.646864:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573962.646867:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573962.646870:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573962.646872:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573962.646875:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573962.646878:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646887:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573962.646890:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646896:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573962.646898:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573962.646902:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573962.646905:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573962.646909:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573962.646911:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305566393
00000100:00000001:0.0:1788573962.646914:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305567161
00000100:00000001:0.0:1788573962.646917:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305566393
00000100:00000001:0.0:1788573962.646920:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305567161
00000100:00000001:0.0:1788573962.646922:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573962.646923:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573962.646925:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573962.646928:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573962/real 1788573962]  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573962.646949:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573962.646950:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573962.646951:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573962.646953:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573962.646955:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573962.646956:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573962.646959:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573962.646965:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646970:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573962.646970:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646971:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573962.646972:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573962.646975:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646979:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573962.646982:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.646987:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573962.646988:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646989:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573962.646990:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.646991:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573962.646993:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573962.646995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573962.646997:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573962.646999:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573962.647002:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573962.647007:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099008:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573962.647010:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.647011:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573962.647013:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393098880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.647017:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573962.647018:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573962.647019:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573962.647021:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573962.647022:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573962.647023:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573962.647025:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573962.647026:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:0.0:1788573962.647028:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573962.647029:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573962.647030:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:0.0:1788573962.647032:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573962.647033:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.647034:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573962.647036:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393099008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573962.647040:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573962.647041:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573962.647042:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573962.647044:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573962.647045:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573962.647045:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573962.647047:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573962.647048:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:0.0:1788573962.647050:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573962.647050:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573962.647051:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:0.0:1788573962.647052:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573962.647053:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.647054:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573962.647057:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573962.647058:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573962.647059:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573962.647060:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573963.029639:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573963.029643:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573963.029646:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573963.157576:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573963.157587:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573963.157590:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573963.157598:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573963.157603:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573963.157608:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573963.157612:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573963.157616:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573963.157622:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573963.605520:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305567161, at 4305567168
00000100:00000040:0.0:1788573963.605524:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305567161, at 4305567168
00000100:00000010:1.0:1788573963.605540:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7cc0.
00000100:00000010:0.0:1788573963.605543:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:1.0:1788573964.181508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573964.181517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573964.181519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573964.181524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573964.181527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573964.181531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573964.181533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573964.181536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573964.181539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573965.077535:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573965.077538:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573965.077541:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573965.205468:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573965.205474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573965.205476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573965.205481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573965.205484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573965.205487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573965.205489:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573965.205491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573965.205493:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573966.101588:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573966.101592:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573966.101595:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573966.229462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573966.229468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573966.229470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573966.229475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573966.229479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573966.229483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573966.229485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573966.229488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573966.229492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573967.253508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573967.253516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573967.253518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573967.253525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573967.253529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573967.253533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573967.253535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573967.253539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573967.253542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573967.765504:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573967.765519:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788573967.765523:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573967.765528:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573967.765535:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573967.765544:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10898
00000400:00000200:1.0:1788573967.765564:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573967.765572:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573967.765576:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573967.765580:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573967.765584:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573967.765587:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573967.765590:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573967.765596:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573967.765599:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788573967.765602:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573967.765606:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788573967.765609:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788573967.765612:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788573967.765615:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573967.765618:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573967.765620:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573967.765622:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573967.765626:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788573967.765630:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573967.765634:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573967.765638:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788573967.765643:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573967.765646:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573967.765649:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788573967.765652:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788573967.765664:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788573967.765668:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788573967.765671:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573967.765673:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573967.765676:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573967.765679:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573967.765684:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573967.765692:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788573967.765701:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573967.765715:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788573967.765728:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573967.765734:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573967.765739:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573967.765764:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788573967.765767:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788573967.765771:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788573967.765772:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573967.765773:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573967.765776:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788573967.765779:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10898
00000400:00000200:1.0:1788573967.765784:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573967.765789:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573967.765792:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573967.765796:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573967.765799:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788573967.765800:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788573967.765802:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788573967.765806:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573967.765807:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788573967.765810:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1788573967.765812:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788573967.765815:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000040:0.0:1788573967.765818:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788573967.765819:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788573967.765824:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788573967.765829:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788573967.765831:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573967.765835:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788573967.765835:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:0.0:1788573967.765839:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:1.0:1788573967.765843:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:0.0:1788573967.765844:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788573967.765847:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1788573967.765849:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788573967.765851:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788573967.765854:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00100000:0.0:1788573967.765856:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393099136:192.168.202.110@tcp:250:kworker.0
00000020:00000040:1.0:1788573967.765857:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788573967.765864:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:1.0:1788573967.765864:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788573967.765867:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573967.765870:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788573967.765870:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788573967.765875:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573967.765877:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1788573967.765879:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:0.0:1788573967.765883:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:1.0:1788573967.765883:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573967.765884:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1788573967.765887:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1788573967.765891:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788573967.765894:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573967.765897:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788573967.765898:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788573967.765903:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000100:00000010:1.0:1788573967.765904:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788573967.765908:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788573967.765910:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573967.765911:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573967.765912:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:0.0:1788573967.765914:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
02000000:00000001:1.0:1788573967.765915:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788573967.765918:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788573967.765923:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000200:0.0:1788573967.765926:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393099136, portal 25
00000100:00000001:1.0:1788573967.765927:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.765933:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573967.765938:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345466752 : -130483364084864 : ffff8953796f4380)
00000100:00100000:1.0:1788573967.765938:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1788573967.765951:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573967.765953:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788573967.765958:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.765968:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788573967.765969:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788573967.765975:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788573967.765985:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788573967.765987:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573967.766001:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393099136, offset 0
00000100:00080000:1.0:1788573967.766004:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788573967.766010:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788573967.766012:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788573967.766015:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788573967.766017:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788573967.766019:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:0.0:1788573967.766023:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788573967.766024:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10898
00000400:00000200:0.0:1788573967.766033:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573967.766041:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573967.766046:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573967.766051:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573967.766054:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573967.766059:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330030:330030:256:4294967295] 192.168.202.110@tcp LPNI seq info [329987:329987:8:4294967295]
00000400:00000200:0.0:1788573967.766065:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573967.766076:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573967.766087:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788573967.766087:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:0.0:1788573967.766095:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000400:00000200:1.0:1788573967.766096:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788573967.766101:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788573967.766104:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:0.0:1788573967.766108:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788573967.766109:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573967.766114:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:0.0:1788573967.766115:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00000001:1.0:1788573967.766117:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788573967.766123:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.1:1788573967.766123:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:1.0:1788573967.766126:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:0.1:1788573967.766128:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:1.0:1788573967.766131:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000100:0.0:1788573967.766134:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:1.0:1788573967.766136:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000010:0.0:1788573967.766138:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000020:00000001:1.0:1788573967.766143:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788573967.766148:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:1.0:1788573967.766149:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788573967.766157:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788573967.766160:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1788573967.766162:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788573967.766167:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788573967.766170:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000100:00000001:1.0:1788573967.766171:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788573967.766177:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788573967.766180:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:0.0:1788573967.766183:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:1.0:1788573967.766184:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788573967.766188:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788573967.766192:0:2361:0:(events.c:35:request_out_callback()) Process entered
02000000:00000001:1.0:1788573967.766193:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788573967.766198:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788573967.766201:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573967.766202:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788573967.766207:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788573967.766214:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766216:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1788573967.766217:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573967.766221:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1788573967.766225:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788573967.766231:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:0.0:1788573967.766232:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766237:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:1.0:1788573967.766238:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788573967.766242:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766243:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573967.766245:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788573967.766248:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1788573967.766251:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788573967.766251:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766254:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305572281
02000000:00000010:1.0:1788573967.766259:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000010:0.0:1788573967.766262:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:1.0:1788573967.766266:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766272:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00100000:1.0:1788573967.766276:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788573967.766281:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305572281
00000100:00000010:0.0:1788573967.766287:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7de0.
00000100:00000001:0.0:1788573967.766291:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788573967.766294:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788573967.766296:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788573967.766299:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766300:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766305:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:1.0:1788573967.766310:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573967.766311:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573967/real 1788573967]  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573967.766341:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1788573967.766342:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573967.766345:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:1.0:1788573967.766347:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1788573967.766348:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573967.766352:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573967.766356:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000040:1.0:1788573967.766356:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:0.0:1788573967.766358:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573967.766364:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788573967.766367:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10909)
00000100:00000200:0.0:1788573967.766372:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766378:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573967.766379:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766381:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573967.766383:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573967.766386:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766409:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573967.766413:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766418:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573967.766420:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766421:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573967.766422:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766425:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573967.766426:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573967.766429:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573967.766432:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573967.766433:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573967.766436:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573967.766442:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393099136:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573967.766447:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766448:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573967.766450:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4380 x1875444393099136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766455:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573967.766457:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573967.766459:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573967.766462:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573967.766463:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573967.766464:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573967.766466:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573967.766469:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:0.0:1788573967.766472:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573967.766474:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573967.766475:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4380.
00000100:00000001:0.0:1788573967.766478:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573967.766479:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766481:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766488:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573967.766489:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766491:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573967.766492:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573967.766493:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573967.766496:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766501:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573967.766502:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573967.766503:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573967.766505:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573967.766506:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573967.766508:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099264:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573967.766510:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573967.766511:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573967.766512:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573967.766514:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573967.766516:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573967.766518:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573967.766521:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573967.766523:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573967.766525:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573967.766527:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393099264, portal 10
00000100:00000001:0.0:1788573967.766529:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573967.766530:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345465856 : -130483364085760 : ffff8953796f4000)
00000100:00000040:0.0:1788573967.766535:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766540:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573967.766541:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573967.766543:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573967.766545:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393099264, offset 0
00000400:00000200:0.0:1788573967.766547:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573967.766550:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573967.766552:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573967.766554:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573967.766556:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573967.766558:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573967.766560:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573967.766562:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330031:330031:256:4294967295] 192.168.202.110@tcp LPNI seq info [329988:329988:8:4294967295]
00000400:00000200:0.0:1788573967.766565:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573967.766570:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573967.766573:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573967.766575:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573967.766577:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573967.766579:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788573967.766582:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573967.766584:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1788573967.766587:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573967.766588:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573967.766591:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573967.766593:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573967.766596:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573967.766597:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573967.766598:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573967.766601:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766606:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573967.766608:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766613:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766614:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573967.766616:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573967.766619:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573967.766621:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573967.766622:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305571367
00000100:00000001:0.0:1788573967.766624:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305572281
00000100:00000001:0.0:1788573967.766626:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305571367
00000100:00000001:0.0:1788573967.766627:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305572281
00000100:00000001:0.0:1788573967.766629:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573967.766630:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766632:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573967.766634:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573967/real 1788573967]  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573967.766638:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573967.766639:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573967.766640:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573967.766642:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573967.766643:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573967.766645:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573967.766647:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573967.766653:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766657:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573967.766658:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766659:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573967.766660:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573967.766662:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766666:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573967.766668:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766672:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573967.766674:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766675:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573967.766676:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766677:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573967.766678:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573967.766680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573967.766682:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573967.766683:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573967.766686:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573967.766693:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099264:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573967.766696:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573967.766700:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766714:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573967.766715:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573967.766718:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573967.766720:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573967.766722:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573967.766726:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099392:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573967.766729:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573967.766732:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573967.766734:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573967.766736:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573967.766740:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788573967.766742:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573967.766746:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573967.766749:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573967.766752:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573967.766755:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393099392, portal 4
00000100:00000001:0.0:1788573967.766757:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573967.766758:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345473920 : -130483364077696 : ffff8953796f5f80)
00000100:00000040:0.0:1788573967.766765:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766771:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573967.766774:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573967.766777:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573967.766779:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393099392, offset 0
00000400:00000200:0.0:1788573967.766784:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573967.766787:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573967.766789:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573967.766792:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573967.766793:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573967.766795:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573967.766797:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573967.766800:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330032:330032:256:4294967295] 192.168.202.110@tcp LPNI seq info [329989:329989:8:4294967295]
00000400:00000200:0.0:1788573967.766803:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573967.766813:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573967.766816:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573967.766818:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573967.766821:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573967.766823:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788573967.766826:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573967.766828:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1788573967.766831:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573967.766832:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573967.766835:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573967.766837:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573967.766839:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573967.766841:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573967.766842:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573967.766845:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766851:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573967.766853:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766857:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766858:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573967.766860:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573967.766863:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573967.766867:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573967.766869:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305571513
00000100:00000001:0.0:1788573967.766872:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305572281
00000100:00000001:0.0:1788573967.766874:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305571513
00000100:00000001:0.0:1788573967.766876:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305572281
00000100:00000001:0.0:1788573967.766879:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573967.766881:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.766884:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573967.766888:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573967/real 1788573967]  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573967.766897:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573967.766899:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573967.766901:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573967.766903:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573967.766905:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573967.766906:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573967.766909:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573967.766919:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766925:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573967.766927:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766929:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573967.766931:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573967.766935:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766943:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573967.766947:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.766956:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573967.766958:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766960:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573967.766961:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766963:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573967.766964:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573967.766966:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573967.766970:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573967.766971:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573967.766974:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573967.766980:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099392:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573967.766983:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.766984:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573967.766986:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4000 x1875444393099264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.767004:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573967.767005:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573967.767007:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573967.767010:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573967.767011:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573967.767012:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573967.767013:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573967.767014:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:0.0:1788573967.767016:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573967.767017:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573967.767018:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4000.
00000100:00000001:0.0:1788573967.767020:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573967.767021:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.767022:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573967.767025:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5f80 x1875444393099392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573967.767029:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573967.767030:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573967.767031:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788573967.767032:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573967.767033:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573967.767034:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573967.767035:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573967.767037:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:0.0:1788573967.767040:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573967.767041:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573967.767042:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5f80.
00000100:00000001:0.0:1788573967.767046:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573967.767047:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.767049:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573967.767052:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573967.767054:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573967.767056:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573967.767058:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573968.149638:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573968.149643:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573968.149646:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573968.277530:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573968.277541:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573968.277544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573968.277550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573968.277554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573968.277561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573968.277564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573968.277568:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573968.277572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573968.725533:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305572281, at 4305572288
00000100:00000040:1.0:1788573968.725533:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305572281, at 4305572288
00000100:00000010:1.0:1788573968.725553:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7de0.
00000100:00000010:0.0:1788573968.725554:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788573969.173676:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573969.173683:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573969.173687:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573969.301533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573969.301541:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573969.301543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573969.301548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573969.301551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573969.301555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573969.301557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573969.301559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573969.301562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573970.325462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573970.325467:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573970.325469:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573970.325474:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573970.325477:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573970.325480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573970.325482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573970.325483:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573970.325486:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573971.221616:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573971.221620:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573971.221624:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573971.349513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573971.349520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573971.349523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573971.349529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573971.349532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573971.349536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573971.349539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573971.349541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573971.349545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573972.245547:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573972.245550:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573972.245553:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788573972.373466:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573972.373473:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573972.373475:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573972.373480:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573972.373483:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573972.373486:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573972.373488:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573972.373491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573972.373493:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573972.885891:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573972.885901:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573972.885905:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573972.885921:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.885948:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573972.885952:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573972.885959:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573972.885962:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573972.885969:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573972.885977:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393099520:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573972.885986:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573972.885990:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573972.885995:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573972.885999:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573972.886011:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573972.886017:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573972.886029:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573972.886035:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573972.886063:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573972.886072:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393099520, portal 25
00000100:00000001:0.0:1788573972.886076:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573972.886079:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000040:0.0:1788573972.886085:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.886094:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573972.886099:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573972.886102:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573972.886109:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393099520, offset 0
00000400:00000200:0.0:1788573972.886129:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573972.886146:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573972.886163:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573972.886186:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573972.886194:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573972.886207:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573972.886219:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573972.886233:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330033:330033:256:4294967295] 192.168.202.110@tcp LPNI seq info [329990:329990:8:4294967295]
00000400:00000200:0.0:1788573972.886253:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573972.886275:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573972.886289:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573972.886302:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573972.886318:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573972.886323:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788573972.886333:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573972.886338:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788573972.886350:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573972.886357:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573972.886370:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573972.886387:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573972.886429:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573972.886432:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573972.886439:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573972.886446:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.886462:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573972.886470:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.886491:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573972.886497:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573972.886507:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573972.886516:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573972.886520:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305577402
00000100:00000010:0.0:1788573972.886528:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573972.886535:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573972.886542:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305577402
00000100:00000010:0.0:1788573972.886546:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d76c0.
00000100:00000001:0.0:1788573972.886551:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573972.886555:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573972.886564:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573972.886568:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573972.886576:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573972/real 1788573972]  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573972.886600:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573972.886605:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573972.886611:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573972.886620:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573972.886630:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573972.886637:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573972.886647:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573972.886681:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.886726:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573972.886732:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.886737:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573972.886742:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573972.886748:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.886758:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573972.886763:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.886768:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573972.886771:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.886772:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573972.886773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.886776:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573972.886778:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573972.886782:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573972.886785:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573972.886786:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573972.886789:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573972.886795:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393099520:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573972.886800:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.886803:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573972.886805:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393099520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.886809:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573972.886812:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573972.886815:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573972.886818:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573972.886821:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573972.886823:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573972.886825:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573972.886828:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788573972.886831:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573972.886833:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573972.886835:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:0.0:1788573972.886839:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573972.886840:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.886842:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.886850:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573972.886852:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573972.886855:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573972.886857:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573972.886858:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573972.886863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.886872:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573972.886873:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573972.886876:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573972.886878:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573972.886881:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573972.886885:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573972.886888:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573972.886891:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573972.886893:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573972.886895:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573972.886898:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573972.886900:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573972.886905:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573972.886908:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573972.886912:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573972.886915:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393099648, portal 10
00000100:00000001:0.0:1788573972.886917:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573972.886919:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:0.0:1788573972.886926:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.886934:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573972.886936:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573972.886939:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573972.886942:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393099648, offset 0
00000400:00000200:0.0:1788573972.886946:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573972.886952:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573972.886956:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573972.886959:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573972.886962:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573972.886964:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573972.886967:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573972.886970:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330034:330034:256:4294967295] 192.168.202.110@tcp LPNI seq info [329991:329991:8:4294967295]
00000400:00000200:0.0:1788573972.886974:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573972.886980:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573972.886985:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573972.886988:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573972.886992:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573972.886995:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788573972.887000:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573972.887003:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788573972.887007:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573972.887009:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573972.887012:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573972.887016:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573972.887019:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573972.887020:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573972.887023:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573972.887027:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887034:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573972.887037:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887043:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573972.887044:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573972.887062:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573972.887066:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573972.887069:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573972.887071:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305576487
00000100:00000001:0.0:1788573972.887073:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305577402
00000100:00000001:0.0:1788573972.887075:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305576487
00000100:00000001:0.0:1788573972.887077:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305577402
00000100:00000001:0.0:1788573972.887079:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573972.887081:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573972.887083:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573972.887087:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573972/real 1788573972]  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573972.887094:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573972.887095:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573972.887097:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573972.887100:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573972.887101:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573972.887103:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573972.887107:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573972.887114:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887120:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573972.887122:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887123:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573972.887125:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573972.887128:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887135:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573972.887139:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887146:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573972.887149:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887150:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573972.887152:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887153:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573972.887156:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573972.887158:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573972.887162:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573972.887164:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573972.887168:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573972.887176:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573972.887180:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573972.887184:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887196:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573972.887197:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573972.887199:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573972.887200:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573972.887202:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573972.887204:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573972.887206:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573972.887208:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573972.887210:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573972.887211:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573972.887213:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788573972.887215:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573972.887218:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573972.887219:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573972.887222:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573972.887224:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393099776, portal 4
00000100:00000001:0.0:1788573972.887225:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573972.887227:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:0.0:1788573972.887230:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887235:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573972.887236:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573972.887238:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573972.887239:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393099776, offset 0
00000400:00000200:0.0:1788573972.887242:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573972.887244:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573972.887247:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573972.887249:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573972.887251:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573972.887253:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573972.887254:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573972.887257:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330035:330035:256:4294967295] 192.168.202.110@tcp LPNI seq info [329992:329992:8:4294967295]
00000400:00000200:0.0:1788573972.887260:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573972.887264:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573972.887267:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573972.887270:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573972.887272:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573972.887274:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788573972.887277:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573972.887279:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788573972.887282:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573972.887283:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788573972.887285:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573972.887287:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573972.887290:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:0.0:1788573972.887291:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:0.0:1788573972.887292:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573972.887295:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887300:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573972.887302:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887307:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573972.887308:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573972.887310:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573972.887312:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573972.887314:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573972.887315:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305576633
00000100:00000001:0.0:1788573972.887317:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305577402
00000100:00000001:0.0:1788573972.887319:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305576633
00000100:00000001:0.0:1788573972.887320:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305577402
00000100:00000001:0.0:1788573972.887322:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573972.887323:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573972.887325:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573972.887327:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573972/real 1788573972]  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573972.887332:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573972.887333:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573972.887334:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573972.887335:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573972.887336:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573972.887338:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573972.887340:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573972.887345:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887349:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573972.887350:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887351:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573972.887352:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573972.887354:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887358:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573972.887360:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887364:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573972.887366:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887367:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573972.887368:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887369:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573972.887370:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573972.887371:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573972.887374:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573972.887375:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573972.887378:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573972.887383:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393099776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573972.887385:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887386:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573972.887388:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393099648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887410:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573972.887412:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573972.887413:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573972.887415:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573972.887416:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573972.887417:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573972.887418:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573972.887419:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788573972.887421:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573972.887422:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573972.887423:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:0.0:1788573972.887425:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573972.887426:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887427:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573972.887429:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393099776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573997 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573972.887434:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573972.887435:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573972.887436:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788573972.887437:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573972.887438:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573972.887439:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573972.887440:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573972.887441:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788573972.887443:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573972.887443:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573972.887444:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:0.0:1788573972.887446:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573972.887446:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887448:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573972.887450:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573972.887451:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573972.887452:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573972.887453:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573973.397492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788573973.397498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788573973.397500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788573973.397505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788573973.397508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788573973.397511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573973.397513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788573973.397516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573973.397518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573973.845498:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305577402, at 4305577408
00000100:00000040:1.0:1788573973.845498:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305577402, at 4305577408
00000100:00000010:0.0:1788573973.845512:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788573973.845517:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d76c0.
00000020:00000001:0.0:1788573974.293592:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573974.293595:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573974.293597:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573975.317535:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573975.317539:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573975.317542:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573977.365557:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573977.365560:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573977.365563:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788573978.005754:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573978.005758:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573978.005761:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573978.005768:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573978.005777:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573978.005780:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573978.005783:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573978.005785:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573978.005788:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573978.005792:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393099904:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573978.005796:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573978.005800:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573978.005803:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573978.005806:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573978.005811:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788573978.005813:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573978.005819:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573978.005823:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573978.005829:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573978.005835:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393099904, portal 25
00000100:00000001:0.0:1788573978.005838:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573978.005840:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:0.0:1788573978.005845:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573978.005854:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573978.005857:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573978.005860:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4088.
00000100:00000200:0.0:1788573978.005863:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393099904, offset 0
00000400:00000200:0.0:1788573978.005869:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573978.005877:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573978.005883:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573978.005889:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573978.005893:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573978.005897:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573978.005900:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573978.005904:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330036:330036:256:4294967295] 192.168.202.110@tcp LPNI seq info [329993:329993:8:4294967295]
00000400:00000200:0.0:1788573978.005910:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573978.005921:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573978.005927:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573978.005933:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788573978.005939:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573978.005945:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788573978.005960:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573978.005965:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573978.005968:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305582521
00000100:00000010:0.0:1788573978.005974:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573978.005982:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573978.005989:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305582521
00000100:00000010:0.0:1788573978.005992:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ba0.
00000100:00000001:0.0:1788573978.005996:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573978.005999:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573978.006002:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573978.006006:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573978.006009:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573978.006016:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573978.006019:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788573978.006021:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573978.006023:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573978.006024:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573978.006029:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573978.006037:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573978.006039:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573978.006042:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573978.006044:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573978.006046:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573978.006048:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100032:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573978.006051:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573978.006052:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573978.006054:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573978.006055:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573978.006058:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788573978.006059:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573978.006062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573978.006064:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dac60.
00000400:00000010:0.0:1788573978.006088:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788573978.006092:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393100032, portal 10
00000100:00000001:0.0:1788573978.006094:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573978.006095:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:0.0:1788573978.006101:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573978.006105:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573978.006107:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573978.006110:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4a18.
00000100:00000200:0.0:1788573978.006111:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393100032, offset 0
00000400:00000200:0.0:1788573978.006115:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573978.006119:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573978.006123:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573978.006125:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573978.006128:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573978.006131:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573978.006133:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573978.006135:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330037:330037:255:4294967295] 192.168.202.110@tcp LPNI seq info [329994:329994:7:4294967295]
00000400:00000200:0.0:1788573978.006138:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573978.006143:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573978.006147:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573978.006150:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788573978.006153:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573978.006156:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788573978.006158:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573978.006161:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573978.006163:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573978.006164:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305581606
00000100:00000001:0.0:1788573978.006166:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305582521
00000100:00000001:0.0:1788573978.006168:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305581606
00000100:00000001:0.0:1788573978.006169:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305582521
00000100:00000001:0.0:1788573978.006171:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573978.006172:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573978.006174:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573978.006177:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573978.006181:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573978.006183:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573978.006184:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573978.006185:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573978.006187:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573978.006188:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100160:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573978.006191:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573978.006192:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573978.006193:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573978.006195:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573978.006197:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573978.006198:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573978.006200:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573978.006201:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788573978.006204:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573978.006206:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393100160, portal 4
00000100:00000001:0.0:1788573978.006207:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573978.006208:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000040:0.0:1788573978.006211:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573978.006216:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573978.006217:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573978.006218:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4bb0.
00000100:00000200:0.0:1788573978.006220:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393100160, offset 0
00000400:00000200:0.0:1788573978.006222:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573978.006224:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573978.006226:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573978.006227:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573978.006229:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573978.006230:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573978.006232:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573978.006234:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330038:330038:254:4294967295] 192.168.202.110@tcp LPNI seq info [329995:329995:6:4294967295]
00000400:00000200:0.0:1788573978.006237:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573978.006241:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573978.006244:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573978.006246:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9300.
00000800:00000200:0.0:1788573978.006248:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573978.006250:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788573978.006251:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573978.006253:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573978.006255:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573978.006256:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305581752
00000100:00000001:0.0:1788573978.006258:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305582521
00000100:00000001:0.0:1788573978.006260:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305581752
00000100:00000001:0.0:1788573978.006262:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305582521
00000100:00000001:0.0:1788573978.006264:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573978.006265:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573978.006268:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573978.006270:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573978.006274:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573978.006275:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788573978.006277:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573978.006278:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573978.006280:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573978.006282:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573978.389536:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573978.389538:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573978.389542:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573978.965496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305582521, at 4305582528
00000100:00000040:0.0:1788573978.965497:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305582521, at 4305582528
00000100:00000010:1.0:1788573978.965513:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ba0.
00000100:00000010:0.0:1788573978.965514:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788573980.437563:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573980.437567:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573980.437570:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:1.0:1788573981.077648:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788573981.077684:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788573981.077694:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573981.077698:0:2352:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788573981.077703:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788573981.077707:0:2352:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788573981.077712:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573981.077716:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:1.0:1788573981.077723:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573981.077728:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573981.077734:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4088
00000400:00000010:1.0:1788573981.077736:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4088.
00000100:00000001:1.0:1788573981.077741:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573981.077751:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573981.077783:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788573981.077790:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574003 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573981.077798:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573981.077800:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788573981.077811:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573981.077814:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:1.0:1788573981.077817:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788573981.077821:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788573981.077825:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4a18
00000400:00000010:1.0:1788573981.077827:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4a18.
00000100:00000001:1.0:1788573981.077831:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573981.077835:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.077841:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788573981.077843:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1788573981.077847:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1788573981.077848:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574003 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.077851:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1788573981.077860:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788573981.077862:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000100:00100000:0.0:1788573981.077863:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573978/real 1788573981]  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574003 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:1.0:1788573981.077873:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788573981.077880:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9300.
00000400:00000200:1.0:1788573981.077887:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000001:0.0:1788573981.077888:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573981.077893:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000200:1.0:1788573981.077893:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000010:0.0:1788573981.077898:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dac60.
00000400:00000200:0.0:1788573981.077905:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788573981.077907:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788573981.077910:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000400:00000200:1.0:1788573981.077913:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4bb0
00000100:00000200:0.0:1788573981.077916:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788573981.077918:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4bb0.
00000100:00000001:1.0:1788573981.077925:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788573981.077931:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788573981.077955:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000200:0.0:1788573981.077959:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788573981.077962:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574003 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.077984:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788573981.077984:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573981.077988:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788573981.077989:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573981.077994:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573981.077998:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573981.078004:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078012:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573981.078019:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078042:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573981.078045:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078047:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573981.078048:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078051:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573981.078055:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573981.078059:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573981.078065:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573981.078067:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573981.078072:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573981.078081:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100032:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573981.078086:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573981.078105:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573978/real 1788573981]  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573981.078113:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573981.078115:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573981.078117:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788573981.078120:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573981.078121:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573981.078124:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573981.078127:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573981.078145:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078153:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573981.078155:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078157:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573981.078158:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573981.078163:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078170:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573981.078174:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078188:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573981.078191:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078193:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573981.078194:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078196:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573981.078198:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573981.078201:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573981.078205:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573981.078208:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573981.078212:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573981.078226:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100160:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573981.078230:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573981.078234:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573978/real 1788573981]  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573981.078243:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573981.078244:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573981.078246:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573981.078249:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573981.078251:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573981.078253:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573981.078256:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573981.078265:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078272:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573981.078273:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078276:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573981.078277:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573981.078281:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078289:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573981.078293:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078301:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573981.078303:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078305:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573981.078306:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078308:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573981.078310:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573981.078313:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573981.078317:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573981.078319:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573981.078324:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573981.078331:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393099904:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573981.078335:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078337:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573981.078341:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393100032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078348:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573981.078351:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573981.078355:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788573981.078359:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573981.078362:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573981.078364:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573981.078367:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573981.078370:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:0.0:1788573981.078374:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573981.078376:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573981.078378:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:0.0:1788573981.078381:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573981.078383:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078385:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573981.078389:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393100160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078445:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573981.078447:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573981.078450:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573981.078453:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573981.078455:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573981.078456:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573981.078459:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573981.078460:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:0.0:1788573981.078463:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573981.078464:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573981.078466:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:0.0:1788573981.078468:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573981.078470:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078472:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573981.078476:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393099904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574003 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573981.078483:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573981.078484:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573981.078486:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788573981.078489:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573981.078490:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573981.078491:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573981.078494:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573981.078495:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:0.0:1788573981.078498:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573981.078499:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573981.078501:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:0.0:1788573981.078504:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573981.078505:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078508:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573981.078517:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573981.078519:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573981.078521:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573981.078523:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573981.461642:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573981.461647:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573981.461650:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788573983.125678:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788573983.125681:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788573983.125699:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788573983.125707:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788573983.125713:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.125749:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573983.125755:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573983.125761:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573983.125772:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.125793:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573983.125797:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573983.125803:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573983.125808:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573983.125813:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573983.125821:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393100288:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573983.125829:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573983.125836:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573983.125841:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573983.125847:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573983.125856:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573983.125867:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573983.125877:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573983.125885:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573983.125896:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573983.125905:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393100288, portal 25
00000100:00000001:0.0:1788573983.125912:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573983.125916:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:0.0:1788573983.125928:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.125942:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573983.125949:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573983.125954:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573983.125958:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393100288, offset 0
00000400:00000200:0.0:1788573983.125970:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573983.125982:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573983.125993:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573983.126002:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573983.126009:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573983.126017:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573983.126025:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573983.126049:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330039:330039:256:4294967295] 192.168.202.110@tcp LPNI seq info [329996:329996:8:4294967295]
00000400:00000200:0.0:1788573983.126062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573983.126079:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573983.126090:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573983.126129:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573983.126142:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573983.126154:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573983.126163:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573983.126169:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788573983.126178:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573983.126181:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573983.126195:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573983.126206:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573983.126214:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573983.126220:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573983.126228:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573983.126237:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126256:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573983.126261:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574008 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126268:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573983.126270:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573983.126276:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573983.126280:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573983.126283:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305587641
00000100:00000010:0.0:1788573983.126288:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573983.126295:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573983.126302:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305587641
00000100:00000010:0.0:1788573983.126305:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7b40.
00000100:00000001:0.0:1788573983.126309:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573983.126320:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573983.126326:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573983.126331:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573983.126339:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573983/real 1788573983]  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573983.126358:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573983.126361:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573983.126366:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573983.126372:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573983.126376:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573983.126382:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573983.126460:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573983.126472:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126478:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573983.126481:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.126484:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573983.126486:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573983.126490:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126498:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573983.126502:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126511:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573983.126514:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.126516:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573983.126517:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.126520:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573983.126522:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573983.126525:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573983.126530:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573983.126532:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573983.126537:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573983.126546:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393100288:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573983.126552:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.126554:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573983.126558:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393100288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126566:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573983.126569:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573983.126572:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573983.126576:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573983.126578:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573983.126580:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573983.126582:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573983.126585:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788573983.126589:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573983.126591:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573983.126593:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:0.0:1788573983.126596:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573983.126597:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.126599:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.126607:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573983.126609:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573983.126611:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573983.126613:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573983.126615:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573983.126619:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126628:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573983.126629:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573983.126632:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573983.126634:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573983.126637:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573983.126640:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100416:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573983.126644:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573983.126646:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573983.126649:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573983.126650:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573983.126654:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573983.126656:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573983.126660:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573983.126663:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573983.126667:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573983.126671:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393100416, portal 10
00000100:00000001:0.0:1788573983.126672:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573983.126674:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:0.0:1788573983.126681:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126689:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573983.126691:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573983.126693:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573983.126696:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393100416, offset 0
00000400:00000200:0.0:1788573983.126700:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573983.126705:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573983.126710:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573983.126713:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573983.126716:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573983.126718:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573983.126720:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573983.126723:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330040:330040:256:4294967295] 192.168.202.110@tcp LPNI seq info [329997:329997:8:4294967295]
00000400:00000200:0.0:1788573983.126727:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573983.126734:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573983.126738:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573983.126741:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573983.126745:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573983.126748:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573983.126753:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573983.126756:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788573983.126760:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573983.126762:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573983.126765:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573983.126769:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573983.126772:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573983.126774:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573983.126776:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573983.126780:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126789:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573983.126792:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574008 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126807:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573983.126808:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573983.126820:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573983.126824:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573983.126830:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573983.126832:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305586727
00000100:00000001:0.0:1788573983.126835:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305587641
00000100:00000001:0.0:1788573983.126838:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305586727
00000100:00000001:0.0:1788573983.126840:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305587641
00000100:00000001:0.0:1788573983.126842:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573983.126845:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573983.126846:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573983.126851:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573983/real 1788573983]  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573983.126858:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573983.126860:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573983.126862:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573983.126864:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573983.126866:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573983.126868:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573983.126872:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573983.126880:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126887:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573983.126889:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.126891:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573983.126892:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573983.126896:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126904:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573983.126908:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126915:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573983.126917:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.126919:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573983.126920:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.126923:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573983.126925:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573983.126927:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573983.126930:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573983.126932:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573983.126937:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573983.126943:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100416:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573983.126948:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573983.126952:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.126958:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573983.126960:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573983.126962:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573983.126963:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573983.126965:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573983.126968:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573983.126971:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573983.126973:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573983.126975:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573983.126977:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573983.126981:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788573983.126982:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573983.126986:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573983.126988:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573983.126992:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573983.126994:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393100544, portal 4
00000100:00000001:0.0:1788573983.126996:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573983.126998:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:0.0:1788573983.127004:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.127010:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573983.127012:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573983.127014:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573983.127016:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393100544, offset 0
00000400:00000200:0.0:1788573983.127020:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573983.127024:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573983.127028:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573983.127031:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573983.127033:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573983.127035:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573983.127038:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573983.127041:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330041:330041:256:4294967295] 192.168.202.110@tcp LPNI seq info [329998:329998:8:4294967295]
00000400:00000200:0.0:1788573983.127045:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573983.127051:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573983.127062:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573983.127066:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573983.127070:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573983.127074:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788573983.127078:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573983.127080:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788573983.127084:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573983.127086:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573983.127089:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573983.127093:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573983.127110:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573983.127112:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573983.127115:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573983.127119:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.127127:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573983.127129:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574008 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.127136:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573983.127138:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573983.127141:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573983.127144:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573983.127147:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573983.127149:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305586873
00000100:00000001:0.0:1788573983.127153:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305587641
00000100:00000001:0.0:1788573983.127155:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305586873
00000100:00000001:0.0:1788573983.127158:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305587641
00000100:00000001:0.0:1788573983.127161:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573983.127163:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573983.127165:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573983.127170:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573983/real 1788573983]  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573983.127176:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573983.127177:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573983.127178:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573983.127180:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573983.127182:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573983.127183:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573983.127186:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573983.127191:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.127195:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573983.127196:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.127197:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573983.127198:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573983.127200:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.127205:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573983.127207:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.127212:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573983.127214:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.127214:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573983.127215:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.127216:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573983.127218:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573983.127219:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573983.127221:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573983.127223:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573983.127225:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573983.127230:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573983.127232:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.127233:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573983.127236:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393100416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.127241:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573983.127242:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573983.127243:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788573983.127245:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573983.127246:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573983.127247:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573983.127248:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573983.127250:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788573983.127252:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573983.127252:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573983.127254:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:0.0:1788573983.127256:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573983.127257:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.127258:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573983.127260:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393100544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574008 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573983.127264:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573983.127265:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573983.127265:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788573983.127267:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573983.127268:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573983.127269:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573983.127270:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573983.127270:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788573983.127272:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573983.127272:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573983.127273:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:0.0:1788573983.127275:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573983.127275:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.127276:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573983.127279:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573983.127280:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573983.127281:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573983.127283:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573983.509616:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573983.509620:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573983.509623:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788573984.085464:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305587641, at 4305587648
00000100:00000040:0.0:1788573984.085466:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305587641, at 4305587648
00000100:00000010:0.0:1788573984.085480:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788573984.085483:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7b40.
00000020:00000001:0.0:1788573984.533564:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573984.533568:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573984.533571:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573986.581544:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573986.581547:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573986.581549:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573987.605579:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573987.605583:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573987.605587:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788573988.245694:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573988.245702:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573988.245706:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573988.245718:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.245743:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573988.245748:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573988.245754:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573988.245757:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573988.245764:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573988.245772:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393100672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573988.245778:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573988.245784:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573988.245787:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573988.245792:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573988.245799:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573988.245803:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573988.245811:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573988.245816:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573988.245824:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573988.245834:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393100672, portal 25
00000100:00000001:0.0:1788573988.245839:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573988.245842:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:0.0:1788573988.245853:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.245883:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573988.245893:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573988.245899:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573988.245906:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393100672, offset 0
00000400:00000200:0.0:1788573988.245921:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573988.245935:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573988.245948:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573988.245956:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573988.245963:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573988.245968:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573988.245971:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573988.245979:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330042:330042:256:4294967295] 192.168.202.110@tcp LPNI seq info [329999:329999:8:4294967295]
00000400:00000200:0.0:1788573988.245989:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573988.246005:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573988.246017:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573988.246025:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573988.246036:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573988.246042:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573988.246053:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573988.246058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788573988.246066:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573988.246069:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573988.246078:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573988.246086:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573988.246093:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573988.246097:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573988.246102:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573988.246109:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246163:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573988.246170:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574013 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246178:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573988.246180:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573988.246186:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573988.246190:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573988.246193:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305592761
00000100:00000010:0.0:1788573988.246199:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573988.246205:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573988.246211:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305592761
00000100:00000010:0.0:1788573988.246214:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7d20.
00000100:00000001:0.0:1788573988.246217:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573988.246220:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573988.246223:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573988.246226:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573988.246230:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573988/real 1788573988]  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573988.246241:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573988.246243:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573988.246245:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573988.246255:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573988.246257:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573988.246260:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573988.246264:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573988.246274:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246282:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573988.246284:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.246287:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573988.246289:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573988.246293:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246302:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573988.246307:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246314:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573988.246318:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.246320:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573988.246321:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.246324:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573988.246327:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573988.246330:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573988.246334:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573988.246336:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573988.246341:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573988.246351:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393100672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573988.246356:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.246358:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573988.246361:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393100672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246368:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573988.246371:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573988.246374:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573988.246377:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573988.246379:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573988.246381:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573988.246383:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573988.246386:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:0.0:1788573988.246389:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573988.246410:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573988.246413:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:0.0:1788573988.246416:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573988.246418:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.246421:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.246427:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573988.246429:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573988.246432:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573988.246433:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573988.246435:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573988.246439:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246447:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573988.246449:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573988.246451:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573988.246453:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573988.246455:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573988.246458:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573988.246462:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573988.246464:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573988.246466:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573988.246468:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573988.246471:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573988.246473:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573988.246477:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573988.246480:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573988.246484:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573988.246487:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393100800, portal 10
00000100:00000001:0.0:1788573988.246489:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573988.246491:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:0.0:1788573988.246497:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246505:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573988.246508:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573988.246510:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573988.246513:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393100800, offset 0
00000400:00000200:0.0:1788573988.246517:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573988.246521:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573988.246524:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573988.246527:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573988.246530:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573988.246533:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573988.246536:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573988.246539:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330043:330043:256:4294967295] 192.168.202.110@tcp LPNI seq info [330000:330000:8:4294967295]
00000400:00000200:0.0:1788573988.246544:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573988.246551:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573988.246557:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573988.246560:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573988.246564:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573988.246567:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573988.246572:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573988.246574:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788573988.246579:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573988.246581:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573988.246584:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573988.246588:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573988.246591:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573988.246593:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573988.246595:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573988.246599:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246608:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573988.246611:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574013 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246626:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573988.246628:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573988.246631:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573988.246635:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573988.246638:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573988.246640:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305591847
00000100:00000001:0.0:1788573988.246643:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305592761
00000100:00000001:0.0:1788573988.246645:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305591847
00000100:00000001:0.0:1788573988.246648:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305592761
00000100:00000001:0.0:1788573988.246650:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573988.246652:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573988.246654:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573988.246658:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573988/real 1788573988]  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573988.246667:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573988.246669:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573988.246670:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573988.246673:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573988.246674:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573988.246677:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573988.246680:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573988.246690:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246697:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573988.246698:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.246700:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573988.246702:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573988.246706:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246713:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573988.246717:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246725:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573988.246728:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.246730:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573988.246732:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.246734:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573988.246736:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573988.246739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573988.246743:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573988.246745:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573988.246750:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573988.246759:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573988.246764:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573988.246768:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246775:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573988.246776:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573988.246778:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573988.246780:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573988.246782:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573988.246786:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100928:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573988.246789:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573988.246791:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573988.246793:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573988.246795:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573988.246798:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788573988.246800:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573988.246804:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573988.246807:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573988.246810:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573988.246813:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393100928, portal 4
00000100:00000001:0.0:1788573988.246815:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573988.246816:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:0.0:1788573988.246822:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246830:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573988.246833:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573988.246835:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573988.246837:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393100928, offset 0
00000400:00000200:0.0:1788573988.246841:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573988.246845:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573988.246848:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573988.246851:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573988.246854:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573988.246856:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573988.246859:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573988.246863:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330044:330044:256:4294967295] 192.168.202.110@tcp LPNI seq info [330001:330001:8:4294967295]
00000400:00000200:0.0:1788573988.246867:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573988.246874:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573988.246879:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573988.246882:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573988.246886:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573988.246889:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788573988.246894:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573988.246897:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788573988.246901:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573988.246904:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573988.246907:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573988.246911:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573988.246914:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573988.246915:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573988.246918:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573988.246921:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246930:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573988.246933:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574013 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.246940:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573988.246942:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573988.246945:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573988.246948:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573988.246951:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573988.246953:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305591993
00000100:00000001:0.0:1788573988.246956:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305592761
00000100:00000001:0.0:1788573988.246958:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305591993
00000100:00000001:0.0:1788573988.246960:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305592761
00000100:00000001:0.0:1788573988.246963:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573988.246965:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573988.246967:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573988.246971:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573988/real 1788573988]  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573988.246979:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573988.246980:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573988.246982:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573988.246984:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573988.246986:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573988.246989:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573988.246992:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573988.247002:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.247009:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573988.247010:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.247012:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573988.247013:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573988.247017:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.247025:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573988.247029:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.247036:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573988.247039:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.247041:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573988.247042:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.247044:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573988.247046:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573988.247049:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573988.247052:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573988.247054:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573988.247059:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573988.247068:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393100928:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573988.247071:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.247074:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573988.247077:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393100800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.247085:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573988.247087:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573988.247089:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788573988.247091:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573988.247093:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573988.247094:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573988.247096:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573988.247098:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:0.0:1788573988.247101:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573988.247102:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573988.247104:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:0.0:1788573988.247107:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573988.247108:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.247110:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573988.247127:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393100928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574013 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573988.247135:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573988.247137:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573988.247138:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788573988.247141:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573988.247142:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573988.247143:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573988.247146:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573988.247147:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788573988.247149:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573988.247151:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573988.247152:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:0.0:1788573988.247155:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573988.247156:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.247158:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573988.247161:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573988.247163:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573988.247164:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573988.247166:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573989.205503:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305592761, at 4305592768
00000100:00000040:1.0:1788573989.205503:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305592761, at 4305592768
00000100:00000010:0.0:1788573989.205518:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788573989.205524:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d20.
00000020:00000001:0.0:1788573989.653619:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573989.653622:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573989.653625:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573990.677602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573990.677605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573990.677608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573992.725549:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573992.725552:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573992.725556:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788573993.365702:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573993.365709:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573993.365714:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573993.365724:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.365746:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573993.365751:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573993.365757:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573993.365763:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573993.365769:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573993.365780:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393101056:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573993.365789:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573993.365795:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573993.365799:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573993.365802:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573993.365810:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573993.365816:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573993.365830:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573993.365837:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573993.365847:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573993.365858:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393101056, portal 25
00000100:00000001:0.0:1788573993.365864:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573993.365868:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:0.0:1788573993.365879:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.365903:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573993.365911:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573993.365919:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573993.365926:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393101056, offset 0
00000400:00000200:0.0:1788573993.365937:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573993.365949:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573993.365957:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573993.365966:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573993.365971:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573993.365976:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573993.365981:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573993.365986:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330045:330045:256:4294967295] 192.168.202.110@tcp LPNI seq info [330002:330002:8:4294967295]
00000400:00000200:0.0:1788573993.365997:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573993.366014:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573993.366024:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573993.366029:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573993.366037:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573993.366044:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573993.366055:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573993.366059:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788573993.366067:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573993.366070:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573993.366077:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573993.366082:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573993.366088:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573993.366091:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573993.366096:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573993.366101:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366112:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573993.366116:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366125:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573993.366126:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573993.366146:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573993.366150:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573993.366152:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305597881
00000100:00000010:0.0:1788573993.366157:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573993.366163:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573993.366167:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305597881
00000100:00000010:0.0:1788573993.366169:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7180.
00000100:00000001:0.0:1788573993.366172:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573993.366176:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573993.366178:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573993.366181:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573993.366186:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573993/real 1788573993]  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573993.366197:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573993.366199:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573993.366201:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573993.366204:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573993.366216:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573993.366219:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573993.366224:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573993.366230:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366235:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573993.366236:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366238:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573993.366239:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573993.366242:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366250:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573993.366256:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366263:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573993.366267:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366269:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573993.366270:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366272:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573993.366274:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573993.366278:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573993.366282:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573993.366284:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573993.366289:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573993.366298:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393101056:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573993.366303:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366305:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573993.366309:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393101056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366315:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573993.366318:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573993.366321:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573993.366325:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573993.366327:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573993.366328:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573993.366331:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573993.366333:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:0.0:1788573993.366337:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573993.366339:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573993.366341:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:0.0:1788573993.366344:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573993.366345:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366347:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366354:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573993.366356:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573993.366358:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573993.366360:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573993.366361:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573993.366365:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366372:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573993.366374:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573993.366376:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573993.366378:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573993.366380:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573993.366383:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101184:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573993.366386:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573993.366388:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573993.366410:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573993.366413:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573993.366417:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573993.366419:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573993.366423:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573993.366426:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573993.366430:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573993.366433:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393101184, portal 10
00000100:00000001:0.0:1788573993.366435:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573993.366437:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:0.0:1788573993.366443:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366448:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573993.366449:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573993.366451:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573993.366453:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393101184, offset 0
00000400:00000200:0.0:1788573993.366456:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573993.366459:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573993.366462:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573993.366464:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573993.366466:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573993.366468:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573993.366470:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573993.366472:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330046:330046:256:4294967295] 192.168.202.110@tcp LPNI seq info [330003:330003:8:4294967295]
00000400:00000200:0.0:1788573993.366475:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573993.366479:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573993.366483:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573993.366485:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573993.366488:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573993.366489:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573993.366493:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573993.366494:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788573993.366497:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573993.366498:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573993.366500:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573993.366503:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573993.366505:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573993.366506:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573993.366508:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573993.366511:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366516:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573993.366518:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366523:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573993.366524:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573993.366526:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573993.366529:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573993.366533:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573993.366535:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305596967
00000100:00000001:0.0:1788573993.366538:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305597881
00000100:00000001:0.0:1788573993.366540:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305596967
00000100:00000001:0.0:1788573993.366543:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305597881
00000100:00000001:0.0:1788573993.366545:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573993.366547:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573993.366549:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573993.366553:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573993/real 1788573993]  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573993.366562:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573993.366563:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573993.366565:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573993.366568:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573993.366570:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573993.366572:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573993.366576:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573993.366585:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366591:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573993.366592:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366594:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573993.366595:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573993.366597:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366602:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573993.366604:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366609:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573993.366611:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366612:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573993.366613:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366614:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573993.366616:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573993.366617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573993.366619:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573993.366620:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573993.366624:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573993.366632:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101184:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573993.366636:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573993.366640:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366647:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573993.366649:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573993.366651:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573993.366653:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573993.366655:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573993.366658:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101312:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573993.366662:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573993.366664:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573993.366666:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573993.366668:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573993.366672:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573993.366674:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573993.366678:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573993.366681:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573993.366685:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573993.366689:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393101312, portal 4
00000100:00000001:0.0:1788573993.366691:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573993.366692:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:0.0:1788573993.366698:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366706:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573993.366708:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573993.366711:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573993.366713:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393101312, offset 0
00000400:00000200:0.0:1788573993.366718:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573993.366722:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573993.366725:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573993.366727:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573993.366729:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573993.366731:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573993.366732:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573993.366735:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330047:330047:256:4294967295] 192.168.202.110@tcp LPNI seq info [330004:330004:8:4294967295]
00000400:00000200:0.0:1788573993.366737:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573993.366744:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573993.366749:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573993.366752:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573993.366756:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573993.366759:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788573993.366764:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573993.366766:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788573993.366770:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573993.366773:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573993.366776:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573993.366780:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573993.366783:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573993.366785:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573993.366788:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573993.366791:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366797:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573993.366799:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366803:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573993.366805:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573993.366808:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573993.366810:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573993.366814:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573993.366817:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305597113
00000100:00000001:0.0:1788573993.366819:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305597881
00000100:00000001:0.0:1788573993.366821:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305597113
00000100:00000001:0.0:1788573993.366824:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305597881
00000100:00000001:0.0:1788573993.366826:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573993.366828:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573993.366830:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573993.366834:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573993/real 1788573993]  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573993.366843:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573993.366845:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573993.366847:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573993.366849:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573993.366851:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573993.366854:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573993.366857:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573993.366863:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366872:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573993.366873:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366874:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573993.366875:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573993.366878:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366882:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573993.366886:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366893:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573993.366896:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366897:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573993.366899:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366901:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573993.366902:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573993.366905:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573993.366909:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573993.366911:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573993.366915:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573993.366925:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101312:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573993.366929:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366931:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573993.366934:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393101184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366938:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573993.366939:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573993.366941:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788573993.366943:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573993.366944:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573993.366945:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573993.366947:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573993.366948:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:0.0:1788573993.366950:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573993.366951:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573993.366952:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:0.0:1788573993.366954:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573993.366955:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366956:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573993.366958:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393101312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573993.366962:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573993.366963:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573993.366964:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788573993.366966:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573993.366967:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573993.366967:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573993.366969:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573993.366970:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788573993.366972:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573993.366973:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573993.366975:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:0.0:1788573993.366977:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573993.366978:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366980:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573993.366984:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573993.366985:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573993.366987:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573993.366989:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573993.749538:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573993.749542:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573993.749546:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788573994.325471:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305597881, at 4305597888
00000100:00000040:1.0:1788573994.325471:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305597881, at 4305597888
00000100:00000010:1.0:1788573994.325486:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7180.
00000100:00000010:0.0:1788573994.325489:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788573995.797550:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573995.797554:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573995.797558:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788573996.821641:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573996.821644:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573996.821648:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788573998.485677:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573998.485682:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573998.485690:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573998.485700:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.485712:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573998.485716:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573998.485721:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573998.485724:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573998.485729:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573998.485736:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393101440:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573998.485743:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573998.485747:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573998.485750:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573998.485754:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573998.485759:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788573998.485763:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573998.485771:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573998.485776:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573998.485784:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573998.485791:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393101440, portal 25
00000100:00000001:0.0:1788573998.485795:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573998.485798:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:0.0:1788573998.485806:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.485817:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573998.485822:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573998.485825:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573998.485829:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393101440, offset 0
00000400:00000200:0.0:1788573998.485838:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573998.485846:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573998.485855:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573998.485861:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573998.485868:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573998.485873:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573998.485878:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573998.485883:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330048:330048:256:4294967295] 192.168.202.110@tcp LPNI seq info [330005:330005:8:4294967295]
00000400:00000200:0.0:1788573998.485890:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573998.485905:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573998.485917:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573998.485925:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573998.485936:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573998.485944:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788573998.485955:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573998.485961:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788573998.485970:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573998.485975:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573998.485981:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573998.485988:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573998.485992:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573998.485994:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573998.486000:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573998.486006:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486018:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573998.486023:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486034:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573998.486038:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573998.486043:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573998.486052:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573998.486057:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305603001
00000100:00000010:0.0:1788573998.486065:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788573998.486074:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788573998.486085:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305603001
00000100:00000010:0.0:1788573998.486091:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7240.
00000100:00000001:0.0:1788573998.486095:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788573998.486103:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573998.486110:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573998.486123:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573998.486133:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573998/real 1788573998]  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573998.486145:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573998.486148:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573998.486163:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573998.486167:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573998.486171:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573998.486175:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573998.486182:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573998.486198:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486216:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573998.486220:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486223:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573998.486226:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573998.486230:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486237:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573998.486241:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486250:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573998.486252:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486254:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573998.486255:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486257:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573998.486259:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573998.486261:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573998.486264:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573998.486266:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573998.486268:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573998.486274:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393101440:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788573998.486277:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486278:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573998.486280:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393101440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486284:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573998.486286:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573998.486288:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788573998.486290:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573998.486291:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573998.486292:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788573998.486294:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573998.486295:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:0.0:1788573998.486298:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573998.486299:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573998.486300:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788573998.486303:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573998.486303:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486305:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486311:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573998.486312:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573998.486314:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573998.486315:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788573998.486316:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573998.486318:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486323:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573998.486324:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573998.486325:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573998.486327:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573998.486328:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573998.486330:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573998.486332:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573998.486333:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573998.486334:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573998.486335:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573998.486337:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788573998.486339:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573998.486341:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573998.486343:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573998.486346:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573998.486348:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393101568, portal 10
00000100:00000001:0.0:1788573998.486349:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573998.486350:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:0.0:1788573998.486355:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486359:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573998.486361:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573998.486363:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573998.486364:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393101568, offset 0
00000400:00000200:0.0:1788573998.486368:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573998.486372:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573998.486376:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573998.486380:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573998.486382:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573998.486384:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573998.486387:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573998.486411:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330049:330049:256:4294967295] 192.168.202.110@tcp LPNI seq info [330006:330006:8:4294967295]
00000400:00000200:0.0:1788573998.486417:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573998.486425:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573998.486428:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573998.486431:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573998.486434:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573998.486436:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788573998.486439:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573998.486441:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788573998.486443:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573998.486444:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573998.486447:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573998.486449:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573998.486451:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573998.486453:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573998.486454:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573998.486457:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486463:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573998.486465:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486470:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573998.486471:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573998.486473:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573998.486475:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788573998.486477:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573998.486479:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305602087
00000100:00000001:0.0:1788573998.486481:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305603001
00000100:00000001:0.0:1788573998.486483:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305602087
00000100:00000001:0.0:1788573998.486484:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305603001
00000100:00000001:0.0:1788573998.486486:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573998.486487:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573998.486489:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573998.486491:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573998/real 1788573998]  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573998.486496:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573998.486497:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573998.486498:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573998.486499:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573998.486500:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573998.486502:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573998.486504:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573998.486509:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486513:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573998.486514:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486515:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573998.486516:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573998.486518:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486522:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573998.486525:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486529:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573998.486531:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486532:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573998.486533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486534:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573998.486536:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573998.486538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573998.486540:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573998.486541:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573998.486544:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573998.486548:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788573998.486551:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788573998.486553:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486557:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788573998.486558:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573998.486560:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788573998.486561:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788573998.486562:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788573998.486564:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573998.486565:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788573998.486567:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788573998.486568:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788573998.486569:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788573998.486571:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573998.486572:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788573998.486574:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788573998.486576:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788573998.486578:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788573998.486580:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393101696, portal 4
00000100:00000001:0.0:1788573998.486581:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788573998.486582:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:0.0:1788573998.486586:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486590:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788573998.486592:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788573998.486593:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788573998.486594:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393101696, offset 0
00000400:00000200:0.0:1788573998.486597:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788573998.486599:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573998.486602:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788573998.486604:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788573998.486606:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788573998.486607:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788573998.486609:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788573998.486611:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330050:330050:256:4294967295] 192.168.202.110@tcp LPNI seq info [330007:330007:8:4294967295]
00000400:00000200:0.0:1788573998.486614:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788573998.486618:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788573998.486621:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573998.486622:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788573998.486625:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788573998.486627:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788573998.486629:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788573998.486631:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788573998.486633:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788573998.486634:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788573998.486636:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788573998.486638:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788573998.486640:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788573998.486641:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788573998.486643:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788573998.486645:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486649:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573998.486651:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486656:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573998.486657:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788573998.486659:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788573998.486660:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788573998.486662:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788573998.486663:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305602233
00000100:00000001:0.0:1788573998.486665:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305603001
00000100:00000001:0.0:1788573998.486667:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305602233
00000100:00000001:0.0:1788573998.486668:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305603001
00000100:00000001:0.0:1788573998.486670:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788573998.486671:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573998.486672:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788573998.486674:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573998/real 1788573998]  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788573998.486678:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788573998.486679:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788573998.486680:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788573998.486681:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788573998.486682:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788573998.486684:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788573998.486685:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788573998.486690:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486695:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788573998.486696:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486697:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573998.486698:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788573998.486700:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486705:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788573998.486707:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486711:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788573998.486713:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486714:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788573998.486714:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486715:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788573998.486717:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788573998.486718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788573998.486720:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788573998.486721:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788573998.486723:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788573998.486728:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788573998.486730:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486731:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573998.486733:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393101568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486737:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573998.486738:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573998.486739:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788573998.486741:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573998.486742:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573998.486743:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788573998.486744:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573998.486746:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:0.0:1788573998.486748:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573998.486748:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573998.486750:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:0.0:1788573998.486751:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573998.486752:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486753:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788573998.486755:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393101696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788573998.486759:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788573998.486760:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788573998.486761:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788573998.486762:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788573998.486763:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788573998.486764:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788573998.486765:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788573998.486766:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788573998.486767:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788573998.486767:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788573998.486768:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:0.0:1788573998.486769:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788573998.486770:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486771:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788573998.486774:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788573998.486774:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788573998.486776:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788573998.486777:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788573998.869655:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573998.869659:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573998.869662:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788573999.445526:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305603001, at 4305603008
00000100:00000040:1.0:1788573999.445526:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305603001, at 4305603008
00000100:00000010:1.0:1788573999.445543:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7240.
00000100:00000010:0.0:1788573999.445546:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788573999.893537:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788573999.893540:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788573999.893544:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574001.941533:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574001.941537:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574001.941540:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574002.965572:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574002.965576:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574002.965580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574003.093518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574003.093528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574003.093531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574003.093539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574003.093543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574003.093548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574003.093552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574003.093557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574003.093561:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574003.605500:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574003.605507:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574003.605509:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574003.605513:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574003.605516:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574003.605529:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10934
00000400:00000200:1.0:1788574003.605538:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574003.605544:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574003.605546:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574003.605548:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605551:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574003.605552:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605554:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574003.605558:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574003.605560:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574003.605561:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574003.605564:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574003.605569:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574003.605570:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574003.605572:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605574:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574003.605575:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605577:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574003.605579:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574003.605581:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605583:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574003.605585:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574003.605588:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574003.605590:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574003.605591:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574003.605593:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574003.605601:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788574003.605603:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574003.605606:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574003.605607:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574003.605609:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574003.605612:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574003.605615:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574003.605621:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574003.605626:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574003.605633:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574003.605641:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574003.605645:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574003.605649:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574003.605667:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574003.605669:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574003.605672:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574003.605673:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574003.605674:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574003.605676:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574003.605677:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10934
00000400:00000200:1.0:1788574003.605679:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574003.605681:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574003.605683:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574003.605685:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605687:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574003.605690:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605692:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574003.605693:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574003.605694:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574003.605696:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574003.605697:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:0.0:1788574003.605699:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1788574003.605699:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788574003.605705:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1788574003.605705:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788574003.605709:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574003.605713:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:0.0:1788574003.605717:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574003.605718:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574003.605725:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605730:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788574003.605732:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574003.605735:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788574003.605736:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:0.0:1788574003.605738:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574003.605742:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1788574003.605744:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1788574003.605745:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574003.605753:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393101824:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574003.605753:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.605757:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574003.605759:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574003.605762:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574003.605764:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574003.605767:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1788574003.605767:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574003.605771:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574003.605777:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:0.0:1788574003.605780:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
00000008:00100000:1.0:1788574003.605781:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788574003.605783:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1788574003.605786:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574003.605789:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574003.605791:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:1.0:1788574003.605795:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000400:00000010:0.0:1788574003.605796:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000020:00000040:1.0:1788574003.605800:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574003.605803:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574003.605804:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574003.605805:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000001:1.0:1788574003.605806:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574003.605810:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574003.605811:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574003.605813:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393101824, portal 25
00000100:00000001:0.0:1788574003.605816:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000010:1.0:1788574003.605816:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:0.0:1788574003.605819:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
00000100:00000001:1.0:1788574003.605822:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574003.605826:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574003.605831:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574003.605838:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574003.605844:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605847:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1788574003.605848:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000001:1.0:1788574003.605850:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574003.605852:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393101824, offset 0
00000100:00000040:1.0:1788574003.605857:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788574003.605864:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605872:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574003.605876:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788574003.605878:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574003.605882:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1788574003.605885:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:1.0:1788574003.605885:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574003.605889:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574003.605892:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:0.0:1788574003.605893:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1788574003.605897:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10934
00000400:00000200:0.0:1788574003.605900:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574003.605904:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574003.605906:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574003.605908:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330051:330051:256:4294967295] 192.168.202.110@tcp LPNI seq info [330008:330008:8:4294967295]
00000400:00000200:0.0:1788574003.605911:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574003.605919:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:1.0:1788574003.605926:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:0.0:1788574003.605927:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574003.605932:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:0.0:1788574003.605933:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000100:00080000:1.0:1788574003.605935:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574003.605940:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:0.0:1788574003.605944:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574003.605945:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1788574003.605952:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:1.0:1788574003.605952:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574003.605957:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574003.605961:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574003.605965:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:0.1:1788574003.605965:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788574003.605970:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:0.1:1788574003.605972:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000040:1.0:1788574003.605974:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574003.605980:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000100:0.0:1788574003.605981:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:1.0:1788574003.605985:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000010:0.0:1788574003.605987:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000100:00000040:1.0:1788574003.605993:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574003.605996:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1788574003.605996:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574003.606000:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574003.606003:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574003.606006:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:0.0:1788574003.606008:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:1.0:1788574003.606012:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574003.606015:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574003.606017:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1788574003.606018:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
02000000:00000001:1.0:1788574003.606019:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574003.606023:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574003.606025:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:1.0:1788574003.606025:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574003.606028:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574003.606031:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606033:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:1.0:1788574003.606033:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574003.606038:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788574003.606041:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000200:0.0:1788574003.606043:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788574003.606044:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574003.606046:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574003.606049:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574003.606052:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574003.606054:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574003.606059:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788574003.606064:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606065:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00100000:1.0:1788574003.606070:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1788574003.606074:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574003.606082:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574003.606086:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574003.606094:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606095:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606101:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574003.606106:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606109:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574003.606110:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574003.606116:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:0.0:1788574003.606118:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574003.606123:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305608121
00000100:00000040:1.0:1788574003.606124:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10944)
00000100:00000010:0.0:1788574003.606133:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574003.606141:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574003.606148:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305608121
00000100:00000010:0.0:1788574003.606152:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7f60.
00000100:00000001:0.0:1788574003.606156:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574003.606160:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574003.606163:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606166:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574003.606183:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574003/real 1788574003]  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574003.606195:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574003.606196:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574003.606198:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574003.606200:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574003.606201:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574003.606204:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574003.606206:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574003.606212:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606217:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574003.606218:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606220:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574003.606221:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574003.606224:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606228:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574003.606231:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606237:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574003.606239:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606240:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574003.606241:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606243:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574003.606244:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574003.606246:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574003.606249:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574003.606251:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574003.606253:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574003.606259:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393101824:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574003.606262:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606264:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574003.606266:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393101824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606270:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574003.606272:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574003.606274:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574003.606276:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574003.606278:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574003.606279:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574003.606280:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574003.606282:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:0.0:1788574003.606285:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574003.606287:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574003.606288:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:0.0:1788574003.606290:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574003.606291:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606292:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606298:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574003.606300:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606301:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574003.606303:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574003.606304:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574003.606308:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606316:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574003.606317:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574003.606319:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574003.606321:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574003.606323:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574003.606327:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101952:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574003.606330:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574003.606332:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574003.606334:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574003.606336:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574003.606339:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574003.606342:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574003.606346:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574003.606349:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574003.606353:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574003.606356:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393101952, portal 10
00000100:00000001:0.0:1788574003.606358:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574003.606360:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:0.0:1788574003.606366:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606371:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574003.606372:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574003.606374:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574003.606375:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393101952, offset 0
00000400:00000200:0.0:1788574003.606378:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574003.606381:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574003.606384:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574003.606386:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574003.606388:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574003.606408:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574003.606410:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574003.606413:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330052:330052:256:4294967295] 192.168.202.110@tcp LPNI seq info [330009:330009:8:4294967295]
00000400:00000200:0.0:1788574003.606415:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574003.606420:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574003.606424:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574003.606426:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574003.606429:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574003.606431:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788574003.606435:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574003.606437:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1788574003.606439:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574003.606440:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574003.606443:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574003.606445:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574003.606448:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574003.606449:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574003.606451:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574003.606453:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606459:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574003.606461:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606465:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606466:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574003.606468:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574003.606471:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574003.606473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574003.606475:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305607207
00000100:00000001:0.0:1788574003.606477:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305608121
00000100:00000001:0.0:1788574003.606478:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305607207
00000100:00000001:0.0:1788574003.606480:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305608121
00000100:00000001:0.0:1788574003.606482:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574003.606483:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606485:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574003.606487:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574003/real 1788574003]  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574003.606492:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574003.606493:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574003.606494:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574003.606496:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574003.606497:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574003.606498:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574003.606500:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574003.606505:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606509:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574003.606510:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606511:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574003.606512:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574003.606514:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606525:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574003.606528:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606532:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574003.606534:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606535:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574003.606536:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606582:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574003.606584:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574003.606586:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574003.606588:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574003.606590:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574003.606593:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574003.606598:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393101952:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574003.606601:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574003.606603:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606607:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574003.606608:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574003.606609:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574003.606611:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574003.606612:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574003.606614:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393102080:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574003.606616:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574003.606617:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574003.606618:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574003.606620:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574003.606622:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574003.606623:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574003.606625:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574003.606627:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574003.606629:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574003.606631:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393102080, portal 4
00000100:00000001:0.0:1788574003.606633:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574003.606634:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00000040:0.0:1788574003.606638:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606642:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574003.606644:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574003.606645:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574003.606647:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393102080, offset 0
00000400:00000200:0.0:1788574003.606650:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574003.606652:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574003.606654:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574003.606656:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574003.606658:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574003.606659:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574003.606661:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574003.606663:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330053:330053:256:4294967295] 192.168.202.110@tcp LPNI seq info [330010:330010:8:4294967295]
00000400:00000200:0.0:1788574003.606666:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574003.606670:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574003.606673:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574003.606675:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574003.606678:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574003.606680:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788574003.606682:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574003.606684:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1788574003.606687:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574003.606688:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574003.606691:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574003.606693:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574003.606695:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574003.606696:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574003.606698:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574003.606707:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606715:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574003.606717:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606726:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606727:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574003.606729:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574003.606731:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574003.606733:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574003.606735:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305607353
00000100:00000001:0.0:1788574003.606737:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305608121
00000100:00000001:0.0:1788574003.606738:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305607353
00000100:00000001:0.0:1788574003.606740:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305608121
00000100:00000001:0.0:1788574003.606741:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574003.606743:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606744:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574003.606747:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574003/real 1788574003]  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574003.606755:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574003.606756:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574003.606758:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574003.606760:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574003.606761:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574003.606764:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574003.606767:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574003.606776:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606783:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574003.606784:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606786:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574003.606788:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574003.606792:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606799:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574003.606803:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606810:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574003.606813:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606815:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574003.606816:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606819:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574003.606820:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574003.606822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574003.606826:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574003.606829:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574003.606832:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574003.606841:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393102080:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574003.606845:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606847:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574003.606850:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393101952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606858:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574003.606859:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574003.606862:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574003.606865:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574003.606867:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574003.606868:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574003.606871:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574003.606872:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:0.0:1788574003.606876:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574003.606877:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574003.606879:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:0.0:1788574003.606881:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574003.606883:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606884:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574003.606889:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393102080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574003.606896:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574003.606897:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574003.606899:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574003.606901:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574003.606902:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574003.606904:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574003.606906:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574003.606907:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:0.0:1788574003.606910:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574003.606911:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574003.606912:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:0.0:1788574003.606915:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574003.606916:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606918:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574003.606922:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574003.606924:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574003.606926:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574003.606927:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574004.117488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574004.117495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574004.117497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574004.117500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574004.117503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574004.117506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574004.117508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574004.117510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574004.117513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574004.565472:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305608121, at 4305608128
00000100:00000040:1.0:1788574004.565472:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305608121, at 4305608128
00000100:00000010:1.0:1788574004.565485:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f60.
00000100:00000010:0.0:1788574004.565487:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788574005.013596:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574005.013600:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574005.013603:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574005.141460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574005.141466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574005.141468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574005.141473:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574005.141476:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574005.141479:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574005.141481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574005.141483:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574005.141486:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574006.037562:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574006.037565:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574006.037568:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574006.165500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574006.165506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574006.165508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574006.165512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574006.165515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574006.165518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574006.165520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574006.165522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574006.165524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574007.189452:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574007.189458:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574007.189460:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574007.189464:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574007.189466:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574007.189469:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574007.189471:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574007.189473:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574007.189475:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574008.085603:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574008.085607:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574008.085610:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574008.213490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574008.213498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574008.213500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574008.213505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574008.213509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574008.213513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574008.213516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574008.213518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574008.213522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574008.725460:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574008.725468:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574008.725470:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574008.725474:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574008.725478:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574008.725484:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10939
00000400:00000200:1.0:1788574008.725491:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574008.725497:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574008.725499:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574008.725502:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725505:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574008.725506:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725508:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574008.725511:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574008.725513:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574008.725515:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574008.725517:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574008.725519:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574008.725520:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574008.725522:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725524:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574008.725525:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725526:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574008.725529:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574008.725531:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725533:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574008.725535:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574008.725538:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574008.725540:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574008.725541:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574008.725543:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574008.725551:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788574008.725553:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574008.725556:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574008.725557:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574008.725559:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574008.725561:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574008.725565:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574008.725571:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574008.725577:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574008.725584:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574008.725597:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574008.725600:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574008.725603:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574008.725622:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574008.725625:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574008.725627:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574008.725628:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574008.725630:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574008.725632:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574008.725634:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10939
00000400:00000200:1.0:1788574008.725636:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574008.725638:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574008.725640:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574008.725642:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725644:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574008.725646:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725647:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574008.725649:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574008.725650:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574008.725651:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574008.725653:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574008.725654:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788574008.725656:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1788574008.725656:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574008.725662:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725663:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788574008.725665:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1788574008.725665:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725666:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788574008.725669:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1788574008.725670:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574008.725674:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725678:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574008.725679:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788574008.725679:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574008.725686:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574008.725690:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.725691:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574008.725692:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574008.725693:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1788574008.725697:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788574008.725698:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:1.0:1788574008.725701:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574008.725702:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574008.725704:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574008.725706:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574008.725709:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000100:00100000:0.0:1788574008.725710:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393102208:192.168.202.110@tcp:250:kworker.0
00000020:00000040:1.0:1788574008.725714:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:0.0:1788574008.725715:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574008.725720:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574008.725721:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574008.725724:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574008.725724:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574008.725730:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788574008.725730:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574008.725734:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:0.0:1788574008.725736:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:1.0:1788574008.725737:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574008.725740:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574008.725745:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000400:00000200:0.0:1788574008.725746:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574008.725753:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574008.725754:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574008.725763:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00100000:1.0:1788574008.725765:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1788574008.725769:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393102208, portal 25
00000100:00000001:0.0:1788574008.725774:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574008.725776:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000001:1.0:1788574008.725783:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1788574008.725785:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574008.725788:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.725797:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574008.725799:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574008.725802:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574008.725805:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574008.725809:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393102208, offset 0
00000400:00000200:0.0:1788574008.725818:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725820:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574008.725824:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788574008.725828:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788574008.725829:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574008.725832:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574008.725835:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1788574008.725836:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:1.0:1788574008.725843:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:0.0:1788574008.725845:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574008.725851:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:1.0:1788574008.725851:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10939
00000400:00000200:0.0:1788574008.725856:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574008.725858:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574008.725861:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330054:330054:256:4294967295] 192.168.202.110@tcp LPNI seq info [330011:330011:8:4294967295]
00000400:00000200:0.0:1788574008.725865:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574008.725872:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574008.725876:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574008.725879:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:0.0:1788574008.725882:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000400:00000200:1.0:1788574008.725885:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574008.725888:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:0.0:1788574008.725892:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788574008.725896:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:0.0:1788574008.725898:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:1.0:1788574008.725899:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574008.725904:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574008.725909:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:0.1:1788574008.725909:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574008.725914:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1788574008.725917:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574008.725919:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000020:00000001:1.0:1788574008.725920:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574008.725923:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574008.725924:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1788574008.725925:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:1.0:1788574008.725926:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574008.725930:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788574008.725931:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:1.0:1788574008.725933:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788574008.725937:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000100:00000040:1.0:1788574008.725938:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:0.0:1788574008.725940:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:1.0:1788574008.725942:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788574008.725946:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:1.0:1788574008.725947:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:0.0:1788574008.725952:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574008.725953:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574008.725960:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:0.0:1788574008.725965:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00080000:1.0:1788574008.725968:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:0.0:1788574008.725971:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574008.725975:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574008.725981:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788574008.725982:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.725985:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:1.0:1788574008.725987:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574008.725990:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574008.725991:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574008.725995:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574008.725995:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574008.725999:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:0.0:1788574008.726000:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305613241
00000008:00000001:1.0:1788574008.726004:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574008.726007:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:1.0:1788574008.726008:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574008.726015:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000100:00000001:0.0:1788574008.726018:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000040:1.0:1788574008.726019:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:0.0:1788574008.726027:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305613241
02000000:00000001:1.0:1788574008.726030:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574008.726032:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574008.726034:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7060.
00000100:00000001:1.0:1788574008.726035:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574008.726037:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574008.726041:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.726042:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000010:1.0:1788574008.726045:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:0.0:1788574008.726048:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574008.726049:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.726056:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574008.726058:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574008.726061:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1788574008.726067:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00100000:0.0:1788574008.726068:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574008/real 1788574008]  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574008.726070:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574008.726078:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574008.726085:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574008.726089:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:1.0:1788574008.726090:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574008.726092:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000100:00000040:1.0:1788574008.726094:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788574008.726096:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000100:00000040:1.0:1788574008.726097:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:0.0:1788574008.726098:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574008.726103:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000040:1.0:1788574008.726103:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10949)
00000100:00000200:0.0:1788574008.726108:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574008.726115:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726126:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574008.726127:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726129:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574008.726130:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574008.726133:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726138:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574008.726141:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726146:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574008.726148:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726149:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574008.726150:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726152:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574008.726154:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574008.726156:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574008.726159:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574008.726161:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574008.726163:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574008.726169:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393102208:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574008.726172:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726173:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574008.726175:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393102208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726179:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574008.726181:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574008.726183:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574008.726199:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574008.726201:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574008.726202:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574008.726204:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574008.726206:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:0.0:1788574008.726208:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574008.726210:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574008.726211:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:0.0:1788574008.726213:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574008.726214:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726216:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726222:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574008.726223:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.726224:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574008.726226:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574008.726227:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574008.726230:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726235:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574008.726236:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574008.726237:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574008.726239:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574008.726240:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574008.726242:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393102336:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574008.726244:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574008.726246:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574008.726247:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574008.726248:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574008.726250:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574008.726252:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574008.726254:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574008.726256:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574008.726259:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574008.726261:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393102336, portal 10
00000100:00000001:0.0:1788574008.726262:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574008.726263:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:0.0:1788574008.726268:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726273:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574008.726274:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574008.726276:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574008.726277:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393102336, offset 0
00000400:00000200:0.0:1788574008.726280:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574008.726283:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574008.726285:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574008.726287:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574008.726290:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574008.726291:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574008.726293:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574008.726295:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330055:330055:256:4294967295] 192.168.202.110@tcp LPNI seq info [330012:330012:8:4294967295]
00000400:00000200:0.0:1788574008.726298:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574008.726303:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574008.726306:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574008.726308:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574008.726311:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574008.726313:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788574008.726316:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574008.726318:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1788574008.726320:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574008.726322:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574008.726324:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574008.726326:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574008.726328:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574008.726329:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574008.726331:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574008.726333:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726339:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574008.726342:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726349:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.726351:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574008.726354:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574008.726358:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574008.726361:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574008.726363:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305612326
00000100:00000001:0.0:1788574008.726366:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305613241
00000100:00000001:0.0:1788574008.726368:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305612326
00000100:00000001:0.0:1788574008.726371:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305613241
00000100:00000001:0.0:1788574008.726373:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574008.726376:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.726378:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574008.726382:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574008/real 1788574008]  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574008.726410:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574008.726412:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574008.726414:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574008.726417:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574008.726419:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574008.726421:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574008.726425:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574008.726434:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726441:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574008.726442:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726444:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574008.726446:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574008.726449:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726457:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574008.726461:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726469:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574008.726471:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726473:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574008.726474:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726476:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574008.726478:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574008.726482:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574008.726485:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574008.726488:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574008.726491:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574008.726500:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393102336:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574008.726504:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574008.726508:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726514:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574008.726516:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574008.726518:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574008.726520:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574008.726522:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574008.726525:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393102464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574008.726529:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574008.726531:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574008.726533:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574008.726534:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574008.726538:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574008.726540:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574008.726544:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574008.726547:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574008.726551:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574008.726554:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393102464, portal 4
00000100:00000001:0.0:1788574008.726556:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574008.726557:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:0.0:1788574008.726563:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726570:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574008.726573:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574008.726576:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574008.726578:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393102464, offset 0
00000400:00000200:0.0:1788574008.726583:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574008.726587:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574008.726590:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574008.726592:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574008.726594:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574008.726596:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574008.726598:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574008.726600:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330056:330056:256:4294967295] 192.168.202.110@tcp LPNI seq info [330013:330013:8:4294967295]
00000400:00000200:0.0:1788574008.726603:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574008.726607:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574008.726611:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574008.726613:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574008.726615:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574008.726617:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788574008.726620:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574008.726622:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1788574008.726625:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574008.726626:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574008.726629:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574008.726631:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574008.726633:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574008.726634:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574008.726636:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574008.726639:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726644:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574008.726646:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726651:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.726652:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574008.726655:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574008.726657:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574008.726659:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574008.726661:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305612473
00000100:00000001:0.0:1788574008.726662:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305613241
00000100:00000001:0.0:1788574008.726664:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305612473
00000100:00000001:0.0:1788574008.726665:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305613241
00000100:00000001:0.0:1788574008.726667:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574008.726668:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.726670:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574008.726672:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574008/real 1788574008]  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574008.726677:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574008.726677:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574008.726678:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574008.726680:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574008.726681:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574008.726682:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574008.726684:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574008.726689:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726693:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574008.726694:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726695:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574008.726696:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574008.726698:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726702:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574008.726705:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726709:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574008.726711:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726712:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574008.726713:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726714:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574008.726715:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574008.726717:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574008.726719:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574008.726720:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574008.726722:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574008.726727:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393102464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574008.726729:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574008.726732:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393102336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726736:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574008.726737:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574008.726739:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574008.726740:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574008.726741:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574008.726742:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574008.726744:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574008.726745:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:0.0:1788574008.726747:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574008.726748:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574008.726749:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:0.0:1788574008.726751:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574008.726752:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726753:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574008.726755:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393102464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574008.726759:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574008.726760:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574008.726761:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574008.726763:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574008.726763:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574008.726764:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574008.726765:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574008.726766:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:0.0:1788574008.726767:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574008.726768:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574008.726769:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:0.0:1788574008.726770:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574008.726771:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726772:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574008.726774:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574008.726775:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574008.726776:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574008.726777:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574009.109584:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574009.109587:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574009.109590:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574009.237472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574009.237479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574009.237482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574009.237488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574009.237491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574009.237495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574009.237498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574009.237501:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574009.237504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574009.685480:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305613241, at 4305613248
00000100:00000040:1.0:1788574009.685480:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305613241, at 4305613248
00000100:00000010:0.0:1788574009.685494:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574009.685497:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7060.
00000100:00000001:1.0:1788574010.261522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574010.261531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574010.261533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574010.261539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574010.261543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574010.261547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574010.261551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574010.261554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574010.261557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574011.157665:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574011.157670:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574011.157674:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574011.285556:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574011.285565:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574011.285568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574011.285573:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574011.285576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574011.285581:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574011.285583:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574011.285586:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574011.285589:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574012.181533:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574012.181537:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574012.181541:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574012.309495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574012.309501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574012.309503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574012.309506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574012.309509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574012.309512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574012.309515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574012.309517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574012.309519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574013.845469:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574013.845476:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574013.845478:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574013.845483:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574013.845487:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574013.845493:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10944
00000400:00000200:0.0:1788574013.845502:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574013.845508:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574013.845511:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574013.845514:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845517:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574013.845520:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845522:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574013.845527:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574013.845529:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574013.845532:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574013.845535:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574013.845538:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574013.845540:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574013.845544:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845546:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574013.845548:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845551:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574013.845554:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574013.845557:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845559:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574013.845561:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574013.845564:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574013.845566:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574013.845568:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574013.845570:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574013.845577:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788574013.845580:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574013.845583:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574013.845584:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574013.845587:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574013.845589:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574013.845592:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574013.845598:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788574013.845604:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574013.845611:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393102592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574013.845619:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574013.845623:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574013.845626:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393102592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574013.845643:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574013.845645:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574013.845648:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574013.845649:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574013.845650:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574013.845652:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574013.845654:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10944
00000400:00000200:0.0:1788574013.845657:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574013.845661:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574013.845663:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574013.845666:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845667:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574013.845669:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845681:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574013.845685:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574013.845686:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574013.845688:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574013.845692:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574013.845696:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574013.845701:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574013.845706:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845709:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574013.845713:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845716:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574013.845720:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574013.845725:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845730:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574013.845733:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574013.845740:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574013.845744:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574013.845747:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574013.845752:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574013.845756:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574013.845760:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574013.845764:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788574013.845768:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574013.845772:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574013.845774:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574013.845777:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574013.845778:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574013.845780:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574013.845787:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574013.845794:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574013.845805:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393102720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574013.845825:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574013.845829:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574013.845838:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393102720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574013.845851:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574013.845855:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574013.845861:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574013.845863:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574013.845864:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574013.845867:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574013.845872:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10944
00000400:00000200:0.0:1788574013.845878:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574013.845884:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574013.845890:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574013.845894:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845898:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574013.845902:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845905:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574013.845910:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574013.845912:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574013.845914:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574013.845918:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574013.845921:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574013.845923:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574013.845928:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845931:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574013.845936:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845938:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574013.845943:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574013.845947:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574013.845952:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574013.845962:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574013.845967:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574013.845971:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574013.845974:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:2.1:1788574013.845976:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000008:00100000:0.0:1788574013.845977:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:2.1:1788574013.845981:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000008:00000001:0.0:1788574013.845982:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574013.845987:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574013.845993:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788574013.845997:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574013.846000:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574013.846002:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574013.846005:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574013.846008:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574013.846010:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574013.846016:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574013.846020:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574013.846027:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393102848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574013.846047:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574013.846051:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574013.846059:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393102848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574013.846071:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574013.846075:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574013.846082:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574013.846087:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10955)
00000800:00000200:2.1:1788574013.846386:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574013.846388:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574013.846672:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574013.846674:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:0.0:1788574014.229563:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574014.229568:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574014.229572:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574014.805472:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305618361, at 4305618368
00000100:00000040:0.0:1788574014.805474:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305618361, at 4305618368
00000100:00000010:1.0:1788574014.805488:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000100:00000010:0.0:1788574014.805494:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000020:00000001:0.0:1788574015.253533:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574015.253537:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574015.253540:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574017.301549:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574017.301552:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574017.301555:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574018.325570:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574018.325573:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574018.325576:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574018.966004:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574018.966009:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574018.966464:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574018.966465:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574018.966673:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574018.966675:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:0.0:1788574019.925499:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305623481, at 4305623488
00000100:00000040:1.0:1788574019.925499:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305623481, at 4305623488
00000100:00000010:1.0:1788574019.925515:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000100:00000010:0.0:1788574019.925518:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000020:00000001:0.0:1788574020.373580:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574020.373584:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574020.373587:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574021.397548:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574021.397551:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574021.397553:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788574022.037499:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574022.037509:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574022.037514:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:0.0:1788574022.037672:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574022.037676:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574022.037693:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574022.037699:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574022.037702:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1788574023.445544:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574023.445547:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574023.445550:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574024.086109:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574024.086115:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574024.086672:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574024.086674:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574024.086889:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574024.086892:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1788574024.469574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574024.469579:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574024.469583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574025.045476:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305628601, at 4305628608
00000100:00000040:1.0:1788574025.045476:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305628601, at 4305628608
00000100:00000010:1.0:1788574025.045493:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000100:00000010:0.0:1788574025.045494:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00000020:00000001:0.0:1788574026.517670:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574026.517674:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574026.517678:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574027.541565:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574027.541568:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574027.541571:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574029.205954:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574029.205960:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788574029.206341:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574029.206343:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788574029.206569:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574029.206571:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1788574029.589567:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574029.589572:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574029.589576:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574030.165490:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305633721, at 4305633728
00000100:00000040:1.0:1788574030.165490:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305633721, at 4305633728
00000100:00000010:1.0:1788574030.165505:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000100:00000010:0.0:1788574030.165507:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00000020:00000001:0.0:1788574030.613555:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574030.613559:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574030.613563:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574031.765498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574031.765504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574031.765506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574031.765511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574031.765515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574031.765518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574031.765526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574031.765533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574031.765536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574032.661613:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574032.661617:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574032.661620:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574032.789491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574032.789499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574032.789501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574032.789508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574032.789511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574032.789515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574032.789518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574032.789520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574032.789523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574033.685590:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574033.685598:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574033.685602:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574033.813550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574033.813559:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574033.813561:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574033.813566:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574033.813570:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574033.813574:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574033.813576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574033.813580:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574033.813583:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574034.326363:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574034.326367:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574034.326761:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574034.326763:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574034.327014:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574034.327017:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:1.0:1788574034.709625:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574034.709631:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574034.709641:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574034.837464:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574034.837474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574034.837476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574034.837483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574034.837487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574034.837491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574034.837494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574034.837497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574034.837500:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574035.285471:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305638842, at 4305638848
00000100:00000010:1.0:1788574035.285491:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000100:00000040:0.0:1788574035.285492:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305638842, at 4305638848
00000100:00000010:0.0:1788574035.285504:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000020:00000001:0.0:1788574035.733620:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574035.733625:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574035.733630:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574035.861515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574035.861523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574035.861525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574035.861530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574035.861533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574035.861536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574035.861538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574035.861541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574035.861543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574036.757573:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574036.757577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574036.757581:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574036.885452:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574036.885459:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574036.885461:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574036.885466:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574036.885470:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574036.885473:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574036.885475:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574036.885478:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574036.885482:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574037.781584:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574037.781588:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574037.781592:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574037.909478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574037.909485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574037.909487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574037.909492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574037.909496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574037.909500:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574037.909503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574037.909506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574037.909509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574038.805584:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574038.805587:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574038.805590:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574038.933496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574038.933504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574038.933508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574038.933513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574038.933518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574038.933527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574038.933531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574038.933534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574038.933538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574039.445544:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574039.445555:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574039.445559:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574039.445562:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574039.445566:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574039.445572:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 10970
00000400:00000200:0.0:1788574039.445580:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574039.445585:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574039.445588:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574039.445590:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445593:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574039.445595:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445596:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574039.445600:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574039.445601:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574039.445604:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574039.445606:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574039.445608:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574039.445610:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574039.445612:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445613:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574039.445614:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445616:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574039.445618:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574039.445620:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445622:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574039.445624:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574039.445627:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574039.445629:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574039.445630:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574039.445632:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574039.445639:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788574039.445642:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574039.445645:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574039.445646:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574039.445649:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574039.445651:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574039.445654:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574039.445661:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574039.445666:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574039.445673:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393104512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574039.445681:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574039.445685:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574039.445689:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393104512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574039.445709:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574039.445711:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574039.445714:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574039.445715:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574039.445716:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574039.445719:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574039.445721:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10970
00000400:00000200:0.0:1788574039.445724:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574039.445726:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574039.445727:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574039.445730:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445731:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574039.445733:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445735:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574039.445736:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574039.445737:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574039.445738:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574039.445740:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574039.445741:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574039.445743:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574039.445745:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445747:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574039.445751:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445753:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574039.445756:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574039.445760:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445764:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574039.445766:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574039.445772:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574039.445776:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574039.445779:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574039.445783:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574039.445789:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574039.445792:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574039.445797:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788574039.445802:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574039.445807:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574039.445809:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574039.445818:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574039.445821:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574039.445825:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574039.445830:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788574039.445834:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574039.445841:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393104640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574039.445854:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574039.445857:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574039.445861:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393104640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574039.445871:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574039.445875:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574039.445879:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574039.445882:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574039.445885:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574039.445888:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574039.445895:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 10970
00000400:00000200:0.0:1788574039.445901:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574039.445907:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574039.445911:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574039.445916:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445920:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574039.445924:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445929:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574039.445930:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574039.445933:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574039.445935:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574039.445936:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574039.445938:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574039.445941:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574039.445947:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445948:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574039.445950:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445951:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574039.445953:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574039.445957:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574039.445961:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574039.445962:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574039.445965:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574039.445969:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574039.445972:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574039.445974:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574039.445977:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574039.445980:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574039.445986:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788574039.445989:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574039.445993:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574039.445994:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574039.445997:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574039.445999:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574039.446002:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574039.446007:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574039.446011:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574039.446018:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393104768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574039.446037:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574039.446040:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574039.446045:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393104768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574039.446054:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574039.446057:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574039.446062:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574039.446065:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10980)
00000800:00000200:2.1:1788574039.446090:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574039.446093:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574039.446494:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574039.446495:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574039.446798:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574039.446799:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:0.0:1788574039.829566:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574039.829569:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574039.829572:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574039.957470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574039.957476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574039.957477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574039.957481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574039.957484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574039.957488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574039.957490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574039.957492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574039.957494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574040.405495:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305643961, at 4305643968
00000100:00000040:1.0:1788574040.405495:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305643961, at 4305643968
00000100:00000010:1.0:1788574040.405509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00000100:00000010:0.0:1788574040.405513:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000020:00000001:1.0:1788574040.853552:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574040.853555:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574040.853557:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574040.981504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574040.981511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574040.981513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574040.981518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574040.981521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574040.981524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574040.981526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574040.981528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574040.981531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574041.877596:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574041.877600:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574041.877604:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574042.005468:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574042.005475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574042.005477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574042.005480:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574042.005483:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574042.005486:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574042.005488:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574042.005491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574042.005493:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574042.901531:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574042.901534:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574042.901537:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574043.029497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574043.029503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574043.029505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574043.029509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574043.029512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574043.029515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574043.029517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574043.029520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574043.029522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574043.925667:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574043.925672:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574043.925676:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574044.053469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574044.053475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574044.053477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574044.053481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574044.053484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574044.053488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574044.053490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574044.053492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574044.053495:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574044.566300:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574044.566799:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574044.567004:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:0.0:1788574044.949587:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574044.949592:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574044.949596:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574045.077458:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574045.077464:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574045.077467:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574045.077472:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574045.077476:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574045.077480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574045.077483:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574045.077486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574045.077489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574045.525496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305649081, at 4305649088
00000100:00000040:0.0:1788574045.525508:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305649081, at 4305649088
00000100:00000010:1.0:1788574045.525509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000100:00000010:0.0:1788574045.525519:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000020:00000001:0.0:1788574045.973581:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574045.973585:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574045.973589:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574046.101488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574046.101494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574046.101496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574046.101500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574046.101503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574046.101506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574046.101508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574046.101511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574046.101512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574046.997604:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574046.997610:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574046.997616:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574047.125501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574047.125509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574047.125511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574047.125517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574047.125521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574047.125525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574047.125528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574047.125531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574047.125534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574047.637853:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574047.637861:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574047.637869:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574047.637882:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574044/real 1788574047]  req@ffff89536d1e1c00 x1875444393104896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574047.637910:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574047.637914:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574047.637921:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119ff20.
00000400:00000200:0.0:1788574047.637929:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65000
00000400:00000010:0.0:1788574047.637935:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65000.
00000100:00000001:0.0:1788574047.637943:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574047.637951:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393104896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574047.637971:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393104896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638001:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574047.638005:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638009:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574047.638014:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574047.638025:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393104896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638038:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574047.638045:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393104896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638053:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574047.638057:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638059:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574047.638060:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638063:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574047.638066:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574047.638070:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574047.638075:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574047.638079:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574047.638083:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393104896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574047.638094:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393104896:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574047.638100:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574047.638103:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574047.638106:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393104896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638113:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574047.638117:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574047.638122:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788574047.638126:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574047.638129:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574047.638131:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574047.638134:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574047.638137:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788574047.638141:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574047.638144:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574047.638146:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:0.0:1788574047.638151:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574047.638153:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638155:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574047.638158:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574047.638160:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574047.638161:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574047.638166:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574044/real 1788574047]  req@ffff89536d1e0380 x1875444393105152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574047.638177:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574047.638179:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574047.638181:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f318.
00000400:00000200:0.0:1788574047.638184:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:0.0:1788574047.638186:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:0.0:1788574047.638188:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574047.638191:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393105152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574047.638208:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393105152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638218:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574047.638219:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638221:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574047.638244:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574047.638249:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393105152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638256:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574047.638260:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393105152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638265:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574047.638280:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638282:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574047.638283:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638285:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574047.638287:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574047.638291:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574047.638295:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574047.638298:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574047.638302:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393105152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574047.638310:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105152:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574047.638314:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574047.638318:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574044/real 1788574047]  req@ffff89536d1e1880 x1875444393105024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574047.638331:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574047.638332:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574047.638335:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f2c0.
00000400:00000200:0.0:1788574047.638338:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65d48
00000400:00000010:0.0:1788574047.638339:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65d48.
00000100:00000001:0.0:1788574047.638342:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574047.638345:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393105024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574047.638353:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393105024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638359:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574047.638361:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638363:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574047.638364:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574047.638368:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393105024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638374:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574047.638378:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393105024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638386:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574047.638388:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638440:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574047.638442:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638445:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574047.638447:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574047.638450:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574047.638455:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574047.638457:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574047.638463:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393105024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574047.638482:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105024:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574047.638488:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638490:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574047.638494:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393105152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638501:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574047.638503:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574047.638505:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788574047.638508:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574047.638510:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574047.638511:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574047.638513:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574047.638515:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788574047.638518:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574047.638520:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574047.638521:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:0.0:1788574047.638524:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574047.638526:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638527:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574047.638531:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393105024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574047.638539:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574047.638542:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574047.638544:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788574047.638548:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574047.638551:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574047.638553:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574047.638554:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574047.638556:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788574047.638559:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574047.638560:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574047.638561:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:0.0:1788574047.638563:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574047.638564:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638566:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574047.638574:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574047.638578:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574047.638581:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574047.638583:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574048.021663:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574048.021670:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574048.021675:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574048.149503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574048.149510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574048.149512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574048.149517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574048.149530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574048.149533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574048.149538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574048.149540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574048.149543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574049.045601:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574049.045605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574049.045607:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574049.173498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574049.173504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574049.173505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574049.173509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574049.173512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574049.173515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574049.173517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574049.173519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574049.173522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.685819:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574049.685832:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574049.685843:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574049.685867:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.685891:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574049.685897:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574049.685906:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574049.685913:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574049.685921:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574049.685933:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393105280:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574049.685950:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574049.685976:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574049.685987:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574049.685998:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574049.686013:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574049.686020:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574049.686044:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574049.686054:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574049.686071:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574049.686085:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393105280, portal 25
00000100:00000001:0.0:1788574049.686094:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574049.686098:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:0.0:1788574049.686117:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.686164:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574049.686176:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574049.686191:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574049.686221:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393105280, offset 0
00000400:00000200:0.0:1788574049.686231:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574049.686244:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574049.686260:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574049.686276:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574049.686293:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574049.686303:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574049.686324:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574049.686341:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330078:330078:256:4294967295] 192.168.202.110@tcp LPNI seq info [330035:330035:8:4294967295]
00000400:00000200:0.0:1788574049.686356:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574049.686382:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574049.686437:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574049.686444:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574049.686452:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574049.686457:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574049.686464:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574049.686469:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574049.686473:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574049.686477:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574049.686483:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574049.686488:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574049.686495:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574049.686498:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574049.686503:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574049.686507:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.686525:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574049.686529:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.686550:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.686552:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574049.686559:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574049.686564:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574049.686567:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305654202
00000100:00000010:0.0:1788574049.686573:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574049.686580:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574049.686591:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305654202
00000100:00000010:0.0:1788574049.686595:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7540.
00000100:00000001:0.0:1788574049.686598:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574049.686602:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574049.686605:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.686608:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574049.686614:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574049/real 1788574049]  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574049.686624:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574049.686627:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574049.686629:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574049.686632:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574049.686634:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574049.686637:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574049.686641:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574049.686650:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.686657:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574049.686660:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.686663:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574049.686666:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574049.686670:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.686683:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574049.686688:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.686696:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574049.686700:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.686702:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574049.686703:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.686706:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574049.686709:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574049.686712:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574049.686717:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574049.686719:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574049.686724:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574049.686733:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393105280:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574049.686739:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.686742:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574049.686746:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393105280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.686753:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574049.686756:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574049.686760:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574049.686765:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574049.686772:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574049.686776:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574049.686782:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574049.686785:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788574049.686788:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574049.686791:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574049.686795:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:0.0:1788574049.686809:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574049.686815:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.686821:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.686839:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574049.686847:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.686854:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574049.686860:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574049.686863:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574049.686876:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.686910:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574049.686917:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574049.686927:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574049.686932:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574049.686939:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574049.686958:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574049.686974:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574049.686984:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574049.686989:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574049.687001:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574049.687008:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574049.687012:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574049.687018:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574049.687022:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574049.687027:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574049.687031:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393105408, portal 10
00000100:00000001:0.0:1788574049.687035:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574049.687039:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:0.0:1788574049.687049:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687058:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574049.687061:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574049.687063:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574049.687066:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393105408, offset 0
00000400:00000200:0.0:1788574049.687071:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574049.687075:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574049.687081:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574049.687084:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574049.687087:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574049.687091:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574049.687093:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574049.687096:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330079:330079:256:4294967295] 192.168.202.110@tcp LPNI seq info [330036:330036:8:4294967295]
00000400:00000200:0.0:1788574049.687100:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574049.687107:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574049.687112:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574049.687115:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574049.687120:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574049.687123:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574049.687127:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574049.687130:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574049.687135:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574049.687137:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574049.687140:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574049.687143:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574049.687147:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574049.687148:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574049.687151:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574049.687154:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687163:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574049.687166:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687178:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.687180:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574049.687183:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574049.687187:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574049.687190:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574049.687192:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305653287
00000100:00000001:0.0:1788574049.687195:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305654202
00000100:00000001:0.0:1788574049.687197:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305653287
00000100:00000001:0.0:1788574049.687199:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305654202
00000100:00000001:0.0:1788574049.687201:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574049.687203:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.687205:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574049.687226:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574049/real 1788574049]  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574049.687234:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574049.687235:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574049.687237:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574049.687240:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574049.687241:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574049.687243:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574049.687246:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574049.687254:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687260:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574049.687262:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687263:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574049.687265:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574049.687268:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687275:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574049.687278:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687286:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574049.687288:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687290:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574049.687292:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687294:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574049.687297:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574049.687299:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574049.687303:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574049.687305:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574049.687309:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574049.687317:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574049.687322:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687324:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574049.687327:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393105408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687333:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574049.687335:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574049.687337:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574049.687339:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574049.687341:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574049.687342:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574049.687345:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574049.687347:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788574049.687350:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574049.687351:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574049.687353:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:0.0:1788574049.687356:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574049.687357:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687359:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687362:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574049.687364:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.687366:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574049.687367:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574049.687369:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574049.687374:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687381:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574049.687383:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574049.687385:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574049.687387:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574049.687418:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574049.687422:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574049.687426:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574049.687429:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574049.687430:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574049.687432:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574049.687436:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574049.687437:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574049.687441:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574049.687444:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574049.687447:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574049.687451:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393105536, portal 4
00000100:00000001:0.0:1788574049.687453:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574049.687455:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:0.0:1788574049.687461:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687466:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574049.687469:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574049.687471:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574049.687474:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393105536, offset 0
00000400:00000200:0.0:1788574049.687478:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574049.687482:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574049.687485:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574049.687488:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574049.687491:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574049.687493:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574049.687496:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574049.687499:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330080:330080:256:4294967295] 192.168.202.110@tcp LPNI seq info [330037:330037:8:4294967295]
00000400:00000200:0.0:1788574049.687503:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574049.687509:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574049.687514:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574049.687518:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574049.687523:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574049.687526:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574049.687530:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574049.687533:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574049.687537:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574049.687539:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574049.687542:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574049.687545:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574049.687549:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574049.687550:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574049.687553:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574049.687557:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687565:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574049.687568:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687574:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.687576:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574049.687579:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574049.687583:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574049.687586:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574049.687588:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305653434
00000100:00000001:0.0:1788574049.687590:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305654202
00000100:00000001:0.0:1788574049.687592:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305653434
00000100:00000001:0.0:1788574049.687594:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305654202
00000100:00000001:0.0:1788574049.687596:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574049.687598:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.687600:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574049.687603:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574049/real 1788574049]  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574049.687611:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574049.687613:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574049.687615:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574049.687617:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574049.687619:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574049.687620:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574049.687623:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574049.687631:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687636:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574049.687638:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687639:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574049.687641:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574049.687644:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687650:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574049.687653:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687660:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574049.687662:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687664:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574049.687665:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687667:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574049.687669:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574049.687671:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574049.687674:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574049.687677:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574049.687681:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574049.687688:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574049.687691:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687694:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574049.687697:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393105536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574049.687704:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574049.687706:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574049.687708:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574049.687711:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574049.687712:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574049.687713:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574049.687715:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574049.687717:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788574049.687720:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574049.687721:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574049.687723:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:0.0:1788574049.687725:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574049.687726:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687728:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574049.687732:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574049.687733:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574049.687735:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574049.687737:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574050.069565:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574050.069569:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574050.069572:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574050.197517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574050.197524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574050.197526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574050.197532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574050.197535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574050.197539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574050.197542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574050.197545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574050.197547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574050.645481:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305654202, at 4305654208
00000100:00000040:1.0:1788574050.645481:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305654202, at 4305654208
00000100:00000010:0.0:1788574050.645496:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574050.645502:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7540.
00000020:00000001:0.0:1788574051.093569:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574051.093573:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574051.093577:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574051.221496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574051.221502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574051.221504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574051.221508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574051.221511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574051.221514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574051.221516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574051.221517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574051.221520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574052.117603:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574052.117606:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574052.117609:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574052.245510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574052.245517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574052.245519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574052.245524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574052.245527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574052.245530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574052.245532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574052.245534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574052.245536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574053.141552:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574053.141555:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574053.141559:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574053.269513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574053.269520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574053.269522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574053.269527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574053.269530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574053.269534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574053.269536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574053.269537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574053.269540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574054.165655:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574054.165659:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574054.165663:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574054.293475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574054.293483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574054.293485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574054.293490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574054.293494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574054.293498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574054.293500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574054.293504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574054.293507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574054.805712:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574054.805718:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574054.805721:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574054.805730:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.805746:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574054.805750:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574054.805756:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574054.805759:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574054.805764:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574054.805767:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393105664:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574054.805774:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574054.805779:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574054.805782:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574054.805785:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574054.805792:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574054.805797:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574054.805806:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574054.805812:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574054.805819:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574054.805825:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393105664, portal 25
00000100:00000001:0.0:1788574054.805830:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574054.805833:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000040:0.0:1788574054.805844:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.805860:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574054.805866:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574054.805869:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574054.805874:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393105664, offset 0
00000400:00000200:0.0:1788574054.805885:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574054.805896:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574054.805906:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574054.805915:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574054.805920:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574054.805926:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574054.805931:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574054.805935:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330081:330081:256:4294967295] 192.168.202.110@tcp LPNI seq info [330038:330038:8:4294967295]
00000400:00000200:0.0:1788574054.805941:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574054.805954:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574054.805960:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574054.805968:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574054.805978:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574054.805983:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1788574054.805991:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574054.805995:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:0.0:1788574054.806002:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574054.806005:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574054.806012:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574054.806021:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574054.806027:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574054.806030:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574054.806037:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574054.806043:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806059:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574054.806064:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574079 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806070:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574054.806072:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574054.806076:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574054.806079:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574054.806081:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305659321
00000100:00000010:0.0:1788574054.806085:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574054.806090:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574054.806095:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305659321
00000100:00000010:0.0:1788574054.806097:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ea0.
00000100:00000001:0.0:1788574054.806100:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574054.806102:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574054.806103:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574054.806105:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574054.806108:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574054/real 1788574054]  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574054.806114:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574054.806115:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574054.806117:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574054.806118:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574054.806120:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574054.806121:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574054.806123:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574054.806129:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806133:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574054.806135:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806137:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574054.806138:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574054.806141:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806145:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574054.806148:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806153:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574054.806155:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806156:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574054.806156:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806158:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574054.806160:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574054.806162:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574054.806164:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574054.806166:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574054.806197:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574054.806203:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393105664:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574054.806207:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806208:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574054.806212:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393105664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806219:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574054.806221:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574054.806224:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574054.806227:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574054.806229:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574054.806231:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574054.806234:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574054.806236:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:0.0:1788574054.806240:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574054.806242:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574054.806244:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:0.0:1788574054.806247:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574054.806248:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806251:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806258:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574054.806260:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574054.806263:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574054.806264:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574054.806265:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574054.806268:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806273:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574054.806274:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574054.806276:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574054.806277:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574054.806279:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574054.806280:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105792:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574054.806283:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574054.806284:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574054.806285:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574054.806287:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574054.806289:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574054.806290:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574054.806293:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574054.806295:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574054.806297:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574054.806300:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393105792, portal 10
00000100:00000001:0.0:1788574054.806301:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574054.806303:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:0.0:1788574054.806307:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806312:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574054.806314:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574054.806315:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574054.806317:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393105792, offset 0
00000400:00000200:0.0:1788574054.806319:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574054.806322:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574054.806324:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574054.806327:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574054.806329:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574054.806331:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574054.806332:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574054.806335:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330082:330082:256:4294967295] 192.168.202.110@tcp LPNI seq info [330039:330039:8:4294967295]
00000400:00000200:0.0:1788574054.806337:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574054.806342:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574054.806345:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574054.806347:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574054.806349:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574054.806351:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1788574054.806354:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574054.806356:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:0.0:1788574054.806358:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574054.806360:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574054.806362:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574054.806364:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574054.806366:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574054.806368:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574054.806370:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574054.806372:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806377:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574054.806379:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574079 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806387:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574054.806389:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574054.806415:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574054.806419:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574054.806421:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574054.806423:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305658407
00000100:00000001:0.0:1788574054.806425:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305659321
00000100:00000001:0.0:1788574054.806427:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305658407
00000100:00000001:0.0:1788574054.806428:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305659321
00000100:00000001:0.0:1788574054.806429:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574054.806431:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574054.806432:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574054.806435:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574054/real 1788574054]  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574054.806440:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574054.806441:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574054.806442:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574054.806443:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574054.806445:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574054.806446:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574054.806448:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574054.806454:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806458:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574054.806459:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806460:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574054.806461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574054.806463:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806468:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574054.806471:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806475:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574054.806477:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806478:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574054.806478:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806480:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574054.806481:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574054.806483:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574054.806485:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574054.806486:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574054.806488:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574054.806493:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105792:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574054.806496:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574054.806498:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806502:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574054.806503:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574054.806504:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574054.806506:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574054.806507:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574054.806508:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105920:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574054.806510:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574054.806512:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574054.806513:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574054.806514:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574054.806516:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574054.806518:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574054.806520:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574054.806522:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574054.806524:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574054.806526:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393105920, portal 4
00000100:00000001:0.0:1788574054.806527:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574054.806528:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:0.0:1788574054.806532:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806536:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574054.806538:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574054.806539:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574054.806540:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393105920, offset 0
00000400:00000200:0.0:1788574054.806543:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574054.806545:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574054.806547:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574054.806549:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574054.806551:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574054.806552:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574054.806554:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574054.806556:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330083:330083:256:4294967295] 192.168.202.110@tcp LPNI seq info [330040:330040:8:4294967295]
00000400:00000200:0.0:1788574054.806559:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574054.806563:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574054.806566:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574054.806568:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574054.806571:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574054.806572:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1788574054.806575:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574054.806577:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:0.0:1788574054.806579:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574054.806581:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574054.806583:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574054.806585:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574054.806587:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574054.806588:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574054.806590:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574054.806592:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806597:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574054.806599:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574079 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806603:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574054.806604:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574054.806606:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574054.806608:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574054.806610:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574054.806611:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305658553
00000100:00000001:0.0:1788574054.806613:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305659321
00000100:00000001:0.0:1788574054.806614:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305658553
00000100:00000001:0.0:1788574054.806615:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305659321
00000100:00000001:0.0:1788574054.806617:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574054.806618:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574054.806619:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574054.806621:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574054/real 1788574054]  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574054.806626:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574054.806626:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574054.806627:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574054.806629:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574054.806630:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574054.806631:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574054.806633:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574054.806639:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806643:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574054.806643:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806644:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574054.806645:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574054.806647:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806651:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574054.806654:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806657:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574054.806659:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806660:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574054.806661:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806662:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574054.806663:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574054.806664:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574054.806666:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574054.806667:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574054.806670:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574054.806675:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393105920:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574054.806677:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806678:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574054.806680:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393105792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806685:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574054.806686:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574054.806687:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574054.806689:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574054.806690:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574054.806691:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574054.806692:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574054.806693:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:0.0:1788574054.806695:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574054.806696:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574054.806697:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:0.0:1788574054.806699:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574054.806700:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806701:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574054.806703:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393105920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574079 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574054.806707:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574054.806708:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574054.806709:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574054.806710:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574054.806711:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574054.806712:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574054.806713:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574054.806714:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788574054.806715:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574054.806715:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574054.806716:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:0.0:1788574054.806718:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574054.806718:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806719:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574054.806721:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574054.806722:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574054.806723:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574054.806725:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574055.189537:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574055.189541:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574055.189544:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574055.317489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574055.317496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574055.317497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574055.317501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574055.317504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574055.317508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574055.317510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574055.317512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574055.317515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574055.765491:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305659321, at 4305659328
00000100:00000040:0.0:1788574055.765499:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305659321, at 4305659328
00000100:00000010:1.0:1788574055.765509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ea0.
00000100:00000010:0.0:1788574055.765511:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788574056.213523:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574056.213526:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574056.213529:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574056.341459:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574056.341468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574056.341471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574056.341477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574056.341481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574056.341484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574056.341486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574056.341489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574056.341491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574057.237565:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574057.237567:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574057.237570:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574057.365492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574057.365498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574057.365500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574057.365504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574057.365506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574057.365509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574057.365511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574057.365513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574057.365515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574058.261547:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574058.261551:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574058.261554:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574058.389479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574058.389486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574058.389488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574058.389492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574058.389495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574058.389498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574058.389499:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574058.389501:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574058.389504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574059.285614:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574059.285618:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574059.285621:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574059.413540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574059.413549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574059.413552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574059.413557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574059.413560:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574059.413564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574059.413567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574059.413569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574059.413572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574059.925509:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574059.925517:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574059.925523:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574059.925783:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574059.925804:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574059.925806:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574059.925822:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574059.925828:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788574059.926005:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574059.926011:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788574059.926455:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574059.926457:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788574059.926694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574059.926696:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:0.0:1788574060.309507:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574060.309511:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574060.309513:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574060.437532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574060.437540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574060.437542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574060.437548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574060.437553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574060.437556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574060.437558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574060.437561:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574060.437563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574060.885529:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305664441, at 4305664448
00000100:00000040:1.0:1788574060.885529:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305664441, at 4305664448
00000100:00000010:0.0:1788574060.885548:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00000100:00000010:1.0:1788574060.885551:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000020:00000001:0.0:1788574061.333573:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574061.333578:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574061.333581:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574061.461457:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574061.461464:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574061.461465:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574061.461471:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574061.461475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574061.461479:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574061.461481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574061.461484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574061.461488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574062.357532:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574062.357537:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574062.357541:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574062.485490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574062.485496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574062.485497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574062.485501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574062.485504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574062.485508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574062.485517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574062.485519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574062.485521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574063.381574:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574063.381578:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574063.381581:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574063.509530:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574063.509542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574063.509545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574063.509553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574063.509557:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574063.509563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574063.509567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574063.509571:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574063.509574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574064.405569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574064.405573:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574064.405576:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574064.533506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574064.533513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574064.533515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574064.533521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574064.533524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574064.533529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574064.533531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574064.533534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574064.533538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574065.045921:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574065.045926:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788574065.046477:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574065.046479:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788574065.046759:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574065.046762:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:1.0:1788574065.429579:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574065.429582:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574065.429585:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574065.557453:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574065.557460:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574065.557461:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574065.557466:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574065.557469:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574065.557473:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574065.557475:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574065.557477:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574065.557479:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574066.005508:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305669561, at 4305669568
00000100:00000040:1.0:1788574066.005508:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305669561, at 4305669568
00000100:00000010:0.0:1788574066.005524:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000100:00000010:1.0:1788574066.005528:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000020:00000001:0.0:1788574066.453558:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574066.453564:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574066.453568:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574066.581510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574066.581518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574066.581520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574066.581526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574066.581530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574066.581534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574066.581537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574066.581540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574066.581543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574067.477695:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574067.477701:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574067.477706:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574068.501575:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574068.501579:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574068.501583:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574069.525688:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574069.525694:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574069.525706:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574070.166192:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574070.166200:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788574070.166875:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574070.166877:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788574070.167096:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574070.167098:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:1.0:1788574070.549593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574070.549599:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574070.549603:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574071.125474:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305674681, at 4305674688
00000100:00000010:1.0:1788574071.125489:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000100:00000040:0.0:1788574071.125498:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305674681, at 4305674688
00000100:00000010:0.0:1788574071.125508:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00000020:00000001:1.0:1788574071.573591:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574071.573594:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574071.573597:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574072.597606:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574072.597610:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574072.597612:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574073.621586:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574073.621589:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574073.621592:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574074.645554:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574074.645558:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574074.645560:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574075.286225:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574075.286232:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788574075.286758:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574075.286760:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788574075.286960:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574075.286962:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:0.0:1788574075.669581:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574075.669590:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574075.669593:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574076.245457:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305679802, at 4305679808
00000100:00000010:0.0:1788574076.245470:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000100:00000040:1.0:1788574076.245554:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305679802, at 4305679808
00000100:00000010:1.0:1788574076.245570:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000020:00000001:1.0:1788574076.693603:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574076.693613:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574076.693616:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574077.717565:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574077.717568:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574077.717571:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574078.741568:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574078.741573:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574078.741577:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574079.765548:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574079.765561:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574079.765566:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574080.406185:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574080.406189:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574080.406587:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574080.406588:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574080.406782:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574080.406784:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:1.0:1788574080.789595:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574080.789600:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574080.789603:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574081.365492:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305684921, at 4305684928
00000100:00000040:1.0:1788574081.365492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305684921, at 4305684928
00000100:00000010:1.0:1788574081.365507:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00000100:00000010:0.0:1788574081.365513:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000020:00000001:0.0:1788574081.813604:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574081.813607:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574081.813611:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574082.837586:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574082.837589:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574082.837592:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574083.861553:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574083.861556:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574083.861558:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574084.885640:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574084.885645:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574084.885649:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574085.526178:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574085.526188:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574085.526751:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574085.526753:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574085.527008:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574085.527010:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788574085.909628:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574085.909632:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574085.909635:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574086.485530:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305690042, at 4305690048
00000100:00000040:1.0:1788574086.485536:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305690042, at 4305690048
00000100:00000010:0.0:1788574086.485554:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000100:00000010:1.0:1788574086.485559:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00000020:00000001:1.0:1788574086.933613:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574086.933617:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574086.933621:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574087.957579:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574087.957583:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574087.957586:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574088.981621:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574088.981626:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574088.981629:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574090.005606:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574090.005610:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574090.005613:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574090.646215:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574090.646224:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574090.646754:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574090.646756:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574090.646969:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574090.646971:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1788574091.029648:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574091.029652:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574091.029655:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574091.605471:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305695161, at 4305695168
00000100:00000040:1.0:1788574091.605486:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305695162, at 4305695168
00000100:00000010:0.0:1788574091.605492:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000100:00000010:1.0:1788574091.605496:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000020:00000001:1.0:1788574092.053598:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574092.053601:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574092.053604:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574093.077542:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574093.077545:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574093.077549:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574094.101518:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574094.101521:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574094.101524:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574095.125550:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574095.125553:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574095.125556:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574095.766023:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574095.766030:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788574095.766470:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574095.766472:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788574095.766665:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574095.766667:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000020:00000001:1.0:1788574096.149566:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574096.149569:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574096.149572:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574096.277534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574096.277543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574096.277546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574096.277551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574096.277554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574096.277557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574096.277560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574096.277562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574096.277565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574096.725472:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305700281, at 4305700288
00000100:00000040:1.0:1788574096.725472:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305700281, at 4305700288
00000100:00000010:0.0:1788574096.725486:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00000100:00000010:1.0:1788574096.725488:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000020:00000001:0.0:1788574097.173613:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574097.173617:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574097.173619:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574097.301470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574097.301476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574097.301478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574097.301482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574097.301485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574097.301488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574097.301490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574097.301492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574097.301494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574097.813501:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574097.813507:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574097.813512:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574097.813654:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574097.813670:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574097.813674:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574097.813676:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:0.0:1788574097.813678:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:1.0:1788574098.197606:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574098.197610:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574098.197613:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574098.325455:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574098.325463:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574098.325465:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574098.325471:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574098.325474:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574098.325478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574098.325481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574098.325485:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574098.325488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574099.221573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574099.221578:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574099.221582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574099.349464:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574099.349470:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574099.349472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574099.349476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574099.349479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574099.349482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574099.349484:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574099.349486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574099.349489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574100.245551:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574100.245556:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574100.245559:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574100.373475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574100.373482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574100.373484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574100.373489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574100.373493:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574100.373498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574100.373501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574100.373504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574100.373507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574100.886234:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574100.886238:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788574100.886777:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574100.886780:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788574100.887160:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574100.887164:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:1.0:1788574101.269547:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574101.269550:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574101.269553:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574101.397497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574101.397502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574101.397504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574101.397508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574101.397511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574101.397513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574101.397515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574101.397517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574101.397519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574101.845502:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305705401, at 4305705408
00000100:00000040:1.0:1788574101.845502:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305705401, at 4305705408
00000100:00000010:1.0:1788574101.845518:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000100:00000010:0.0:1788574101.845521:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000020:00000001:1.0:1788574102.293555:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574102.293559:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574102.293562:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574102.421462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574102.421468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574102.421470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574102.421474:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574102.421477:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574102.421480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574102.421482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574102.421484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574102.421486:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574103.317569:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574103.317573:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574103.317576:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574103.445486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574103.445493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574103.445494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574103.445499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574103.445503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574103.445506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574103.445508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574103.445510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574103.445512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574104.341551:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574104.341555:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574104.341558:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574104.469461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574104.469468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574104.469470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574104.469476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574104.469479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574104.469483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574104.469486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574104.469489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574104.469492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574105.365627:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574105.365631:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574105.365635:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574105.493544:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574105.493554:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574105.493565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574105.493571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574105.493575:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574105.493580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574105.493582:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574105.493585:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574105.493588:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574106.005703:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574106.005719:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574106.005722:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574106.005732:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.005748:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574106.005751:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574106.005757:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574106.005762:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574106.005766:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574106.005770:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393109504:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574106.005775:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574106.005781:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574106.005784:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574106.005789:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574106.005795:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574106.005799:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574106.005809:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574106.005815:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574106.005824:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574106.005833:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393109504, portal 25
00000100:00000001:0.0:1788574106.005838:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574106.005843:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:0.0:1788574106.005852:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.005871:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574106.005877:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574106.005881:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574106.005886:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393109504, offset 0
00000400:00000200:0.0:1788574106.005895:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574106.005907:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574106.005917:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574106.005924:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574106.005930:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574106.005936:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574106.005941:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574106.005947:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330111:330111:256:4294967295] 192.168.202.110@tcp LPNI seq info [330068:330068:8:4294967295]
00000400:00000200:0.0:1788574106.005959:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574106.005975:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574106.005985:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574106.005993:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574106.006004:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574106.006009:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574106.006021:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574106.006026:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574106.006034:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574106.006037:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574106.006045:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574106.006052:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574106.006057:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574106.006059:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574106.006063:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574106.006066:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006075:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574106.006078:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006084:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574106.006085:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574106.006090:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574106.006093:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574106.006095:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305710521
00000100:00000010:0.0:1788574106.006100:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574106.006106:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574106.006110:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305710521
00000100:00000010:0.0:1788574106.006112:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7120.
00000100:00000001:0.0:1788574106.006114:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574106.006116:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574106.006118:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574106.006120:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574106.006123:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574106/real 1788574106]  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574106.006130:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574106.006131:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574106.006133:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574106.006135:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574106.006136:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574106.006138:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574106.006140:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574106.006146:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006150:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574106.006152:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006154:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574106.006155:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574106.006158:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006162:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574106.006165:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006170:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574106.006171:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006172:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574106.006173:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006175:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574106.006176:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574106.006178:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574106.006181:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574106.006182:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574106.006185:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574106.006190:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393109504:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574106.006193:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006195:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574106.006197:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393109504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006202:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574106.006203:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574106.006205:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574106.006207:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574106.006208:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574106.006210:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574106.006211:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574106.006213:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788574106.006216:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574106.006217:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574106.006219:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:0.0:1788574106.006221:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574106.006222:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006224:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006229:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574106.006231:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574106.006232:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574106.006233:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574106.006234:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574106.006237:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006242:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574106.006243:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574106.006244:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574106.006246:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574106.006247:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574106.006249:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393109632:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574106.006251:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574106.006252:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574106.006253:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574106.006254:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574106.006256:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574106.006257:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574106.006260:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574106.006261:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574106.006263:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574106.006266:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393109632, portal 10
00000100:00000001:0.0:1788574106.006267:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574106.006268:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:0.0:1788574106.006272:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006277:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574106.006279:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574106.006280:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574106.006282:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393109632, offset 0
00000400:00000200:0.0:1788574106.006284:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574106.006286:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574106.006289:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574106.006291:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574106.006292:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574106.006294:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574106.006296:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574106.006298:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330112:330112:256:4294967295] 192.168.202.110@tcp LPNI seq info [330069:330069:8:4294967295]
00000400:00000200:0.0:1788574106.006301:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574106.006305:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574106.006308:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574106.006310:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574106.006313:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574106.006315:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574106.006318:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574106.006319:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574106.006322:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574106.006323:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574106.006326:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574106.006328:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574106.006330:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574106.006331:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574106.006332:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574106.006335:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006340:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574106.006342:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006350:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574106.006351:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574106.006353:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574106.006356:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574106.006358:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574106.006359:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305709606
00000100:00000001:0.0:1788574106.006361:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305710521
00000100:00000001:0.0:1788574106.006363:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305709606
00000100:00000001:0.0:1788574106.006364:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305710521
00000100:00000001:0.0:1788574106.006365:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574106.006367:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574106.006368:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574106.006370:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574106/real 1788574106]  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574106.006375:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574106.006376:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574106.006377:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574106.006378:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574106.006379:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574106.006381:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574106.006383:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574106.006388:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006434:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574106.006436:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006437:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574106.006438:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574106.006440:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006446:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574106.006449:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006453:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574106.006455:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006456:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574106.006457:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006467:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574106.006469:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574106.006470:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574106.006473:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574106.006475:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574106.006477:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574106.006483:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393109632:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574106.006485:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574106.006487:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006492:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574106.006493:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574106.006494:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574106.006495:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574106.006497:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574106.006499:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393109760:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574106.006500:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574106.006502:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574106.006503:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574106.006504:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574106.006507:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574106.006508:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574106.006511:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574106.006513:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574106.006515:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574106.006517:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393109760, portal 4
00000100:00000001:0.0:1788574106.006524:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574106.006526:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:0.0:1788574106.006532:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006537:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574106.006538:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574106.006540:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574106.006542:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393109760, offset 0
00000400:00000200:0.0:1788574106.006544:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574106.006547:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574106.006549:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574106.006551:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574106.006553:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574106.006554:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574106.006556:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574106.006558:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330113:330113:256:4294967295] 192.168.202.110@tcp LPNI seq info [330070:330070:8:4294967295]
00000400:00000200:0.0:1788574106.006561:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574106.006565:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574106.006568:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574106.006570:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574106.006572:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574106.006574:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574106.006577:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574106.006579:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574106.006582:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574106.006583:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574106.006585:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574106.006588:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574106.006590:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4330
00000400:00000010:0.0:1788574106.006591:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4330.
00000100:00000001:0.0:1788574106.006592:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574106.006595:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006600:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574106.006602:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006606:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574106.006607:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574106.006609:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574106.006611:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574106.006613:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574106.006614:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305709753
00000100:00000001:0.0:1788574106.006616:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305710521
00000100:00000001:0.0:1788574106.006618:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305709753
00000100:00000001:0.0:1788574106.006619:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305710521
00000100:00000001:0.0:1788574106.006621:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574106.006622:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574106.006623:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574106.006625:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574106/real 1788574106]  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574106.006631:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574106.006632:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574106.006634:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574106.006636:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574106.006638:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574106.006640:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574106.006644:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574106.006652:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006659:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574106.006660:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006662:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574106.006664:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574106.006668:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006676:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574106.006680:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006688:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574106.006691:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006692:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574106.006693:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006695:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574106.006696:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574106.006698:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574106.006700:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574106.006702:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574106.006705:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574106.006726:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393109760:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574106.006729:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574106.006733:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393109632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006738:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574106.006739:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574106.006740:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574106.006742:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574106.006743:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574106.006744:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574106.006746:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574106.006747:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574106.006749:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574106.006750:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574106.006751:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:0.0:1788574106.006752:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574106.006753:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006755:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574106.006757:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393109760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574131 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574106.006760:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574106.006761:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574106.006762:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574106.006763:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574106.006764:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574106.006764:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574106.006765:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574106.006766:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788574106.006768:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574106.006768:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574106.006769:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:0.0:1788574106.006770:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574106.006771:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006772:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574106.006775:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574106.006775:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574106.006777:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574106.006778:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574106.389566:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574106.389570:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574106.389573:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574106.517515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574106.517523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574106.517525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574106.517530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574106.517534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574106.517539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574106.517542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574106.517545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574106.517548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574106.965469:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305710521, at 4305710528
00000100:00000040:1.0:1788574106.965469:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305710521, at 4305710528
00000100:00000010:1.0:1788574106.965486:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7120.
00000100:00000010:0.0:1788574106.965491:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788574107.413563:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574107.413566:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574107.413569:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574107.541499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574107.541514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574107.541517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574107.541522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574107.541525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574107.541529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574107.541532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574107.541535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574107.541538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574108.437562:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574108.437565:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574108.437568:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574108.565512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574108.565520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574108.565522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574108.565528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574108.565532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574108.565537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574108.565540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574108.565543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574108.565545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574109.461567:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574109.461570:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574109.461573:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574109.589505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574109.589513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574109.589515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574109.589520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574109.589524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574109.589528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574109.589530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574109.589533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574109.589536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574110.485632:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574110.485636:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574110.485639:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574110.613470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574110.613479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574110.613481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574110.613487:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574110.613491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574110.613494:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574110.613496:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574110.613499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574110.613502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574111.125807:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574111.125812:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574111.125817:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574111.125826:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393109888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574111.125844:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574111.125847:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574111.125852:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574111.125855:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574111.125859:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574111.125868:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393109888:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574111.125879:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574111.125885:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574111.125889:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574111.125895:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574111.125904:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574111.125908:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574111.125918:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574111.125924:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574111.125931:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574111.125938:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393109888, portal 25
00000100:00000001:0.0:1788574111.125942:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574111.125944:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:0.0:1788574111.125952:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393109888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574111.126007:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574111.126014:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574111.126019:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4330.
00000100:00000200:0.0:1788574111.126022:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393109888, offset 0
00000400:00000200:0.0:1788574111.126033:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574111.126045:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574111.126054:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574111.126062:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574111.126070:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574111.126076:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574111.126082:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574111.126092:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330114:330114:256:4294967295] 192.168.202.110@tcp LPNI seq info [330071:330071:8:4294967295]
00000400:00000200:0.0:1788574111.126100:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574111.126123:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574111.126134:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574111.126140:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574111.126148:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574111.126167:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574111.126198:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574111.126208:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574111.126213:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305715641
00000100:00000010:0.0:1788574111.126220:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574111.126229:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574111.126251:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305715641
00000100:00000010:0.0:1788574111.126259:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7120.
00000100:00000001:0.0:1788574111.126263:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574111.126269:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574111.126275:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574111.126281:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574111.126284:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574111.126292:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574111.126296:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788574111.126299:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574111.126301:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574111.126304:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574111.126311:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393110016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574111.126322:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574111.126326:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574111.126331:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574111.126333:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574111.126337:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574111.126345:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110016:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574111.126353:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574111.126356:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574111.126359:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574111.126361:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574111.126366:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574111.126370:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574111.126377:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574111.126384:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574111.126389:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574111.126393:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393110016, portal 10
00000100:00000001:0.0:1788574111.126395:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574111.126396:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:0.0:1788574111.126402:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393110016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574111.126461:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574111.126463:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574111.126467:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d47f8.
00000100:00000200:0.0:1788574111.126469:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393110016, offset 0
00000400:00000200:0.0:1788574111.126472:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574111.126482:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574111.126502:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574111.126506:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574111.126510:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574111.126512:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574111.126514:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574111.126517:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330115:330115:255:4294967295] 192.168.202.110@tcp LPNI seq info [330072:330072:7:4294967295]
00000400:00000200:0.0:1788574111.126520:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574111.126525:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574111.126528:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574111.126530:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9400.
00000800:00000200:0.0:1788574111.126533:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574111.126536:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574111.126538:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574111.126540:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574111.126543:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574111.126545:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305714727
00000100:00000001:0.0:1788574111.126547:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305715641
00000100:00000001:0.0:1788574111.126549:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305714727
00000100:00000001:0.0:1788574111.126550:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305715641
00000100:00000001:0.0:1788574111.126564:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574111.126566:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574111.126569:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574111.126570:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574111.126572:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574111.126574:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788574111.126575:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574111.126576:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574111.126577:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574111.126580:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393110144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574111.126585:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574111.126603:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574111.126604:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574111.126607:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574111.126608:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574111.126610:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574111.126612:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574111.126614:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574111.126615:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574111.126616:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574111.126618:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574111.126626:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574111.126630:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574111.126632:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dac60.
00000400:00000010:0.0:1788574111.126636:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574111.126639:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393110144, portal 4
00000100:00000001:0.0:1788574111.126641:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574111.126642:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:0.0:1788574111.126649:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393110144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574111.126656:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574111.126659:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574111.126662:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4c38.
00000100:00000200:0.0:1788574111.126664:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393110144, offset 0
00000400:00000200:0.0:1788574111.126669:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574111.126674:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574111.126678:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574111.126680:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574111.126682:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574111.126684:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574111.126686:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574111.126688:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330116:330116:254:4294967295] 192.168.202.110@tcp LPNI seq info [330073:330073:6:4294967295]
00000400:00000200:0.0:1788574111.126691:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574111.126696:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574111.126701:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574111.126705:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9900.
00000800:00000200:0.0:1788574111.126709:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574111.126713:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574111.126716:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574111.126718:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574111.126722:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574111.126724:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305714873
00000100:00000001:0.0:1788574111.126726:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305715641
00000100:00000001:0.0:1788574111.126728:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305714873
00000100:00000001:0.0:1788574111.126730:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305715641
00000100:00000001:0.0:1788574111.126732:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574111.126734:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574111.126737:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574111.126739:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574111.126742:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574111.126744:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788574111.126746:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574111.126747:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574111.126748:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574111.126750:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574111.509535:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574111.509538:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574111.509556:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574111.637444:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574111.637450:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574111.637452:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574111.637456:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574111.637459:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574111.637462:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574111.637464:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574111.637466:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574111.637469:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574112.085491:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305715641, at 4305715648
00000100:00000040:0.0:1788574112.085495:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305715641, at 4305715648
00000100:00000010:1.0:1788574112.085506:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7120.
00000100:00000010:0.0:1788574112.085511:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788574112.533558:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574112.533564:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574112.533567:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574112.661455:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574112.661461:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574112.661462:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574112.661466:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574112.661469:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574112.661472:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574112.661474:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574112.661476:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574112.661478:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574113.557558:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574113.557561:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574113.557564:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574113.685526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574113.685533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574113.685535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574113.685540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574113.685543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574113.685546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574113.685548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574113.685551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574113.685553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574114.197850:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574114.197857:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574114.197863:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574114.197876:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574111/real 1788574114]  req@ffff895348e00e00 x1875444393109888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574114.197901:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574114.197905:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574114.197910:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574114.197919:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574114.197924:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574114.197933:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574114.197943:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393109888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574114.197965:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393109888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.197986:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574114.197991:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.197996:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574114.198002:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574114.198013:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393109888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198025:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574114.198032:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393109888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198041:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574114.198044:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198046:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574114.198048:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198051:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574114.198053:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574114.198058:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574114.198064:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574114.198066:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574114.198071:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393109888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574114.198080:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393109888:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574114.198084:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574114.198088:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574114.198091:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393109888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198097:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574114.198101:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574114.198106:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574114.198111:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574114.198115:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574114.198117:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574114.198120:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574114.198122:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788574114.198126:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574114.198128:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574114.198131:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:0.0:1788574114.198135:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574114.198136:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198138:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574114.198141:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574114.198142:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574114.198143:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574114.198148:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574111/real 1788574114]  req@ffff895348e00a80 x1875444393110144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574114.198157:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574114.198159:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574114.198162:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dac60.
00000400:00000200:0.0:1788574114.198164:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574114.198166:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574114.198168:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574114.198171:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393110144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574114.198181:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393110144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198188:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574114.198189:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198192:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574114.198194:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574114.198198:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393110144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198206:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574114.198210:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393110144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198226:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574114.198230:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198231:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574114.198233:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198235:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574114.198237:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574114.198240:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574114.198244:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574114.198246:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574114.198250:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393110144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574114.198257:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574114.198262:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574114.198266:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574111/real 1788574114]  req@ffff895348e02d80 x1875444393110016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574114.198274:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574114.198275:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574114.198277:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574114.198280:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574114.198281:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574114.198284:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574114.198288:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393110016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574114.198295:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393110016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198301:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574114.198303:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198304:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574114.198306:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574114.198309:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393110016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198315:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574114.198318:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393110016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198325:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574114.198327:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198328:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574114.198329:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198331:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574114.198333:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574114.198335:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574114.198339:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574114.198341:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574114.198345:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393110016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574114.198368:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110016:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574114.198372:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198374:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574114.198377:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393110144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198384:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574114.198386:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574114.198388:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574114.198441:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574114.198443:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574114.198444:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574114.198447:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574114.198449:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788574114.198452:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574114.198454:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574114.198455:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:0.0:1788574114.198470:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574114.198471:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198474:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574114.198478:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393110016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574136 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574114.198486:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574114.198487:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574114.198489:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574114.198492:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574114.198493:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574114.198495:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574114.198497:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574114.198498:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788574114.198500:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574114.198502:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574114.198503:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:0.0:1788574114.198506:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574114.198507:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198509:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574114.198517:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574114.198520:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574114.198522:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574114.198523:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574114.581578:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574114.581588:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574114.581591:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574114.709525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574114.709531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574114.709533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574114.709537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574114.709540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574114.709544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574114.709546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574114.709548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574114.709550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574115.605554:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574115.605559:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574115.605563:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574115.733493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574115.733502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574115.733505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574115.733510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574115.733514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574115.733518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574115.733521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574115.733524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574115.733527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574116.245710:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574116.245717:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574116.245722:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574116.245734:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.245750:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574116.245754:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574116.245760:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574116.245765:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574116.245771:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574116.245781:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393110272:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574116.245791:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574116.245798:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574116.245801:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574116.245805:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574116.245815:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574116.245820:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574116.245831:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574116.245838:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574116.245848:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574116.245861:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393110272, portal 25
00000100:00000001:0.0:1788574116.245867:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574116.245871:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:0.0:1788574116.245885:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.245900:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574116.245923:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574116.245929:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574116.245937:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393110272, offset 0
00000400:00000200:0.0:1788574116.245950:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574116.245961:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574116.245986:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574116.245997:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574116.246005:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574116.246013:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574116.246020:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574116.246031:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330117:330117:256:4294967295] 192.168.202.110@tcp LPNI seq info [330074:330074:8:4294967295]
00000400:00000200:0.0:1788574116.246043:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574116.246065:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574116.246081:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574116.246091:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574116.246106:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574116.246114:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574116.246128:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574116.246134:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574116.246148:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574116.246153:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574116.246165:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574116.246175:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574116.246186:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574116.246192:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574116.246200:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574116.246209:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246231:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574116.246239:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574141 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246258:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574116.246262:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574116.246272:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574116.246278:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574116.246284:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305720761
00000100:00000010:0.0:1788574116.246290:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574116.246300:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574116.246308:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305720761
00000100:00000010:0.0:1788574116.246312:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ea0.
00000100:00000001:0.0:1788574116.246315:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574116.246319:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574116.246322:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574116.246324:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574116.246330:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574116/real 1788574116]  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574116.246340:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574116.246342:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574116.246345:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574116.246348:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574116.246350:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574116.246352:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574116.246356:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574116.246365:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246371:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574116.246373:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.246375:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574116.246377:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574116.246382:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246389:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574116.246394:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246437:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574116.246441:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.246443:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574116.246447:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.246450:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574116.246452:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574116.246456:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574116.246460:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574116.246464:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574116.246468:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574116.246477:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393110272:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574116.246483:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.246486:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574116.246489:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393110272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246497:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574116.246499:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574116.246502:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574116.246506:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574116.246508:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574116.246510:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574116.246512:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574116.246515:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788574116.246519:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574116.246521:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574116.246523:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:0.0:1788574116.246526:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574116.246528:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.246530:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.246540:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574116.246543:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574116.246545:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574116.246546:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574116.246548:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574116.246552:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246560:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574116.246561:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574116.246563:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574116.246565:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574116.246568:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574116.246571:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574116.246575:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574116.246577:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574116.246579:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574116.246581:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574116.246585:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574116.246587:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574116.246591:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574116.246595:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574116.246599:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574116.246602:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393110400, portal 10
00000100:00000001:0.0:1788574116.246604:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574116.246606:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:0.0:1788574116.246613:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246621:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574116.246623:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574116.246625:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574116.246628:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393110400, offset 0
00000400:00000200:0.0:1788574116.246633:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574116.246637:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574116.246641:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574116.246644:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574116.246646:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574116.246649:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574116.246651:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574116.246655:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330118:330118:256:4294967295] 192.168.202.110@tcp LPNI seq info [330075:330075:8:4294967295]
00000400:00000200:0.0:1788574116.246659:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574116.246666:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574116.246670:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574116.246674:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574116.246678:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574116.246681:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574116.246686:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574116.246688:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574116.246692:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574116.246694:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574116.246697:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574116.246701:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574116.246704:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574116.246705:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574116.246708:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574116.246712:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246719:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574116.246722:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574141 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246728:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574116.246730:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574116.246733:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574116.246737:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574116.246740:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574116.246743:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305719847
00000100:00000001:0.0:1788574116.246746:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305720761
00000100:00000001:0.0:1788574116.246748:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305719847
00000100:00000001:0.0:1788574116.246750:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305720761
00000100:00000001:0.0:1788574116.246752:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574116.246755:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574116.246757:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574116.246760:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574116/real 1788574116]  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574116.246769:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574116.246770:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574116.246772:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574116.246775:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574116.246777:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574116.246779:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574116.246781:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574116.246789:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246796:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574116.246797:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.246799:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574116.246801:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574116.246805:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246812:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574116.246816:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246823:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574116.246826:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.246828:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574116.246829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.246831:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574116.246833:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574116.246836:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574116.246840:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574116.246842:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574116.246846:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574116.246855:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574116.246859:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574116.246863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246870:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574116.246872:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574116.246874:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574116.246876:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574116.246878:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574116.246881:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110528:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574116.246884:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574116.246886:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574116.246888:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574116.246889:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574116.246894:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574116.246896:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574116.246899:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574116.246902:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574116.246906:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574116.246910:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393110528, portal 4
00000100:00000001:0.0:1788574116.246912:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574116.246913:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:0.0:1788574116.246920:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.246928:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574116.246931:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574116.246933:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574116.246936:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393110528, offset 0
00000400:00000200:0.0:1788574116.246940:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574116.246945:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574116.246949:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574116.246952:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574116.246955:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574116.246958:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574116.246960:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574116.246964:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330119:330119:256:4294967295] 192.168.202.110@tcp LPNI seq info [330076:330076:8:4294967295]
00000400:00000200:0.0:1788574116.246969:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574116.246975:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574116.246980:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574116.246985:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574116.246989:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574116.246992:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574116.246996:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574116.246999:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574116.247003:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574116.247006:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574116.247009:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574116.247013:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574116.247016:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574116.247018:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574116.247021:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574116.247025:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.247034:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574116.247036:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574141 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.247041:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574116.247042:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574116.247045:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574116.247048:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574116.247050:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574116.247053:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305719993
00000100:00000001:0.0:1788574116.247056:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305720761
00000100:00000001:0.0:1788574116.247058:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305719993
00000100:00000001:0.0:1788574116.247060:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305720761
00000100:00000001:0.0:1788574116.247063:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574116.247065:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574116.247067:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574116.247071:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574116/real 1788574116]  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574116.247079:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574116.247080:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574116.247083:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574116.247085:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574116.247087:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574116.247089:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574116.247093:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574116.247102:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.247108:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574116.247112:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.247114:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574116.247115:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574116.247132:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.247140:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574116.247144:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.247149:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574116.247151:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.247152:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574116.247153:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.247154:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574116.247155:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574116.247157:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574116.247159:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574116.247160:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574116.247163:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574116.247169:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110528:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574116.247171:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.247173:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574116.247175:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393110400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.247179:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574116.247180:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574116.247182:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574116.247188:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574116.247190:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574116.247191:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574116.247192:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574116.247194:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788574116.247195:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574116.247197:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574116.247198:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:0.0:1788574116.247200:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574116.247201:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.247202:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574116.247204:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393110528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574141 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574116.247209:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574116.247209:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574116.247210:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788574116.247212:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574116.247212:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574116.247213:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574116.247214:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574116.247215:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788574116.247216:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574116.247217:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574116.247218:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:0.0:1788574116.247219:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574116.247220:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.247221:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574116.247225:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574116.247226:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574116.247227:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574116.247228:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574116.629582:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574116.629587:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574116.629591:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574116.757451:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574116.757458:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574116.757460:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574116.757464:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574116.757471:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574116.757474:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574116.757479:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574116.757481:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574116.757484:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574117.205462:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305720761, at 4305720768
00000100:00000040:1.0:1788574117.205470:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305720761, at 4305720768
00000100:00000010:0.0:1788574117.205476:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574117.205480:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ea0.
00000020:00000001:1.0:1788574117.653539:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574117.653542:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574117.653544:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574117.781496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574117.781503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574117.781506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574117.781511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574117.781517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574117.781521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574117.781524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574117.781527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574117.781531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574118.677554:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574118.677557:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574118.677560:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574118.805517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574118.805523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574118.805525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574118.805529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574118.805532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574118.805534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574118.805536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574118.805538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574118.805540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574119.701578:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574119.701582:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574119.701585:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574119.829482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574119.829490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574119.829491:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574119.829497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574119.829500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574119.829503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574119.829505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574119.829507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574119.829509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574120.725563:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574120.725568:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574120.725571:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574120.853497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574120.853504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574120.853505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574120.853509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574120.853512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574120.853514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574120.853516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574120.853518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574120.853520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574121.365673:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574121.365679:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574121.365685:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574121.365697:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.365714:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574121.365719:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574121.365726:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574121.365731:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574121.365738:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574121.365748:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393110656:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574121.365757:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574121.365764:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574121.365769:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574121.365774:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574121.365781:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574121.365785:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574121.365797:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574121.365803:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574121.365814:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574121.365824:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393110656, portal 25
00000100:00000001:0.0:1788574121.365831:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574121.365836:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:0.0:1788574121.365849:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.365870:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574121.365877:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574121.365889:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574121.365897:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393110656, offset 0
00000400:00000200:0.0:1788574121.365908:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574121.365920:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574121.365930:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574121.365941:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574121.365949:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574121.365957:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574121.365965:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574121.365974:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330120:330120:256:4294967295] 192.168.202.110@tcp LPNI seq info [330077:330077:8:4294967295]
00000400:00000200:0.0:1788574121.365988:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574121.366008:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574121.366019:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574121.366026:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574121.366034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574121.366038:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574121.366044:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574121.366048:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574121.366053:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574121.366056:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574121.366062:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574121.366067:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574121.366072:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574121.366075:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574121.366079:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574121.366084:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366094:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574121.366098:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366105:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574121.366107:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574121.366113:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574121.366117:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574121.366120:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305725881
00000100:00000010:0.0:1788574121.366126:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574121.366133:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574121.366139:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305725881
00000100:00000010:0.0:1788574121.366142:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7540.
00000100:00000001:0.0:1788574121.366145:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574121.366148:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574121.366151:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574121.366154:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574121.366159:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574121/real 1788574121]  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574121.366169:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574121.366171:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574121.366174:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574121.366176:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574121.366178:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574121.366181:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574121.366185:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574121.366194:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366201:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574121.366203:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366206:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574121.366208:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574121.366212:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366220:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574121.366225:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366233:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574121.366236:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366237:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574121.366254:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366258:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574121.366260:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574121.366263:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574121.366267:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574121.366270:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574121.366276:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574121.366284:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393110656:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574121.366290:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366292:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574121.366296:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393110656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366303:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574121.366305:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574121.366308:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574121.366311:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574121.366313:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574121.366315:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574121.366318:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574121.366320:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788574121.366324:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574121.366326:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574121.366328:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:0.0:1788574121.366331:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574121.366332:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366334:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366341:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574121.366343:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574121.366346:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574121.366347:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574121.366349:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574121.366353:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366360:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574121.366362:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574121.366364:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574121.366366:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574121.366368:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574121.366372:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110784:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574121.366376:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574121.366378:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574121.366379:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574121.366382:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574121.366385:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574121.366387:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574121.366415:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574121.366418:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574121.366423:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574121.366427:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393110784, portal 10
00000100:00000001:0.0:1788574121.366429:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574121.366430:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530777728 : -130484178773888 : ffff895348e01880)
00000100:00000040:0.0:1788574121.366438:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366445:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574121.366448:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574121.366450:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574121.366453:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393110784, offset 0
00000400:00000200:0.0:1788574121.366458:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574121.366462:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574121.366466:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574121.366469:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574121.366472:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574121.366475:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574121.366477:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574121.366481:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330121:330121:256:4294967295] 192.168.202.110@tcp LPNI seq info [330078:330078:8:4294967295]
00000400:00000200:0.0:1788574121.366485:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574121.366492:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574121.366497:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574121.366500:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574121.366505:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574121.366508:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574121.366513:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574121.366516:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574121.366520:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574121.366522:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574121.366526:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574121.366529:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574121.366532:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574121.366534:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574121.366537:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574121.366541:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366549:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574121.366552:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366560:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574121.366561:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574121.366564:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574121.366568:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574121.366572:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574121.366574:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305724967
00000100:00000001:0.0:1788574121.366577:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305725881
00000100:00000001:0.0:1788574121.366580:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305724967
00000100:00000001:0.0:1788574121.366582:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305725881
00000100:00000001:0.0:1788574121.366584:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574121.366587:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574121.366588:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574121.366592:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574121/real 1788574121]  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574121.366601:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574121.366602:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574121.366604:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574121.366607:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574121.366609:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574121.366611:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574121.366615:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574121.366624:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366630:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574121.366632:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366634:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574121.366635:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574121.366639:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366646:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574121.366650:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366657:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574121.366660:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366662:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574121.366663:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366665:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574121.366667:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574121.366670:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574121.366674:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574121.366676:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574121.366681:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574121.366689:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110784:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574121.366693:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574121.366697:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366713:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574121.366714:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574121.366717:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574121.366719:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574121.366721:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574121.366724:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110912:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574121.366728:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574121.366730:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574121.366732:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574121.366734:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574121.366737:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574121.366739:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574121.366743:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574121.366745:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574121.366749:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574121.366751:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393110912, portal 4
00000100:00000001:0.0:1788574121.366753:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574121.366754:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:0.0:1788574121.366758:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366762:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574121.366764:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574121.366765:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574121.366767:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393110912, offset 0
00000400:00000200:0.0:1788574121.366769:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574121.366772:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574121.366775:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574121.366777:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574121.366779:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574121.366780:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574121.366782:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574121.366785:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330122:330122:256:4294967295] 192.168.202.110@tcp LPNI seq info [330079:330079:8:4294967295]
00000400:00000200:0.0:1788574121.366787:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574121.366792:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574121.366795:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574121.366797:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574121.366800:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574121.366802:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574121.366804:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574121.366806:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574121.366810:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574121.366811:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574121.366813:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574121.366815:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574121.366817:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574121.366818:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574121.366820:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574121.366822:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366827:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574121.366829:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366834:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574121.366835:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574121.366837:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574121.366839:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574121.366841:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574121.366842:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305725113
00000100:00000001:0.0:1788574121.366844:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305725881
00000100:00000001:0.0:1788574121.366845:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305725113
00000100:00000001:0.0:1788574121.366847:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305725881
00000100:00000001:0.0:1788574121.366848:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574121.366850:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574121.366851:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574121.366854:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574121/real 1788574121]  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574121.366859:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574121.366860:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574121.366861:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574121.366862:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574121.366863:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574121.366864:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574121.366866:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574121.366872:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366876:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574121.366876:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366878:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574121.366878:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574121.366881:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366884:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574121.366887:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366891:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574121.366892:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366893:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574121.366894:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366895:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574121.366896:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574121.366898:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574121.366900:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574121.366902:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574121.366904:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574121.366909:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393110912:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574121.366911:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366913:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574121.366915:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01880 x1875444393110784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366918:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574121.366920:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574121.366921:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574121.366922:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574121.366923:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574121.366924:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574121.366925:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574121.366927:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788574121.366928:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574121.366929:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574121.366930:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01880.
00000100:00000001:0.0:1788574121.366932:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574121.366933:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366934:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574121.366936:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393110912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574121.366940:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574121.366941:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574121.366942:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574121.366943:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574121.366943:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574121.366944:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574121.366945:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574121.366947:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788574121.366948:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574121.366949:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574121.366950:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:0.0:1788574121.366951:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574121.366952:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366953:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574121.366956:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574121.366957:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574121.366958:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574121.366959:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574121.749556:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574121.749559:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574121.749562:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574121.877454:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574121.877461:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574121.877463:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574121.877467:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574121.877470:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574121.877473:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574121.877475:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574121.877477:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574121.877480:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574122.325454:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305725881, at 4305725888
00000100:00000040:0.0:1788574122.325460:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305725881, at 4305725888
00000100:00000010:1.0:1788574122.325468:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7540.
00000100:00000010:0.0:1788574122.325471:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788574122.773520:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574122.773524:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574122.773528:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574122.901493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574122.901500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574122.901501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574122.901505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574122.901508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574122.901511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574122.901513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574122.901515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574122.901517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574123.797543:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574123.797546:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574123.797549:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574123.925481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574123.925486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574123.925488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574123.925492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574123.925495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574123.925498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574123.925500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574123.925502:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574123.925504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574124.821573:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574124.821576:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574124.821580:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574124.949515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574124.949522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574124.949524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574124.949529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574124.949533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574124.949536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574124.949539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574124.949541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574124.949543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574125.845569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574125.845575:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574125.845579:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574125.973484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574125.973495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574125.973499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574125.973507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574125.973512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574125.973517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574125.973520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574125.973524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574125.973528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574126.485771:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574126.485777:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574126.485780:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574126.485789:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.485802:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574126.485804:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574126.485809:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574126.485812:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574126.485815:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574126.485821:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393111040:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574126.485827:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574126.485832:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574126.485835:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574126.485838:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574126.485844:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574126.485849:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574126.485859:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574126.485863:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574126.485869:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574126.485877:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393111040, portal 25
00000100:00000001:0.0:1788574126.485881:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574126.485884:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:0.0:1788574126.485894:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.485903:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574126.485909:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574126.485913:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574126.485918:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393111040, offset 0
00000400:00000200:0.0:1788574126.485926:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574126.485935:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574126.485942:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574126.485949:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574126.485956:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574126.485964:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574126.485969:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574126.485975:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330123:330123:256:4294967295] 192.168.202.110@tcp LPNI seq info [330080:330080:8:4294967295]
00000400:00000200:0.0:1788574126.485982:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574126.485996:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574126.486004:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574126.486010:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574126.486017:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574126.486033:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574126.486044:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574126.486051:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574126.486064:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574126.486069:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574126.486080:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574126.486103:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574126.486124:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574126.486128:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574126.486134:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574126.486141:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486170:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574126.486177:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486185:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574126.486188:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574126.486194:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574126.486201:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574126.486205:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305731001
00000100:00000010:0.0:1788574126.486213:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574126.486222:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574126.486232:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305731001
00000100:00000010:0.0:1788574126.486234:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7060.
00000100:00000001:0.0:1788574126.486237:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574126.486239:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574126.486241:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574126.486243:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574126.486248:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574126/real 1788574126]  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574126.486255:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574126.486257:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574126.486259:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574126.486261:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574126.486262:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574126.486264:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574126.486266:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574126.486272:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486276:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574126.486278:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486279:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574126.486281:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574126.486283:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486288:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574126.486291:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486295:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574126.486297:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486298:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574126.486299:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486300:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574126.486302:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574126.486304:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574126.486307:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574126.486308:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574126.486311:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574126.486316:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393111040:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574126.486319:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486321:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574126.486323:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393111040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486327:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574126.486329:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574126.486332:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574126.486334:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574126.486335:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574126.486336:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574126.486338:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574126.486340:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788574126.486343:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574126.486344:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574126.486345:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:0.0:1788574126.486347:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574126.486348:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486350:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486356:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574126.486358:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574126.486361:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574126.486362:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574126.486364:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574126.486368:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486375:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574126.486377:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574126.486379:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574126.486381:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574126.486383:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574126.486386:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393111168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574126.486390:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574126.486419:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574126.486421:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574126.486423:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574126.486427:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574126.486429:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574126.486433:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574126.486436:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574126.486440:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574126.486444:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393111168, portal 10
00000100:00000001:0.0:1788574126.486446:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574126.486447:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:0.0:1788574126.486452:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486457:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574126.486458:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574126.486460:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574126.486461:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393111168, offset 0
00000400:00000200:0.0:1788574126.486464:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574126.486467:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574126.486470:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574126.486472:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574126.486474:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574126.486475:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574126.486477:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574126.486479:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330124:330124:256:4294967295] 192.168.202.110@tcp LPNI seq info [330081:330081:8:4294967295]
00000400:00000200:0.0:1788574126.486482:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574126.486487:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574126.486490:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574126.486492:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574126.486495:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574126.486497:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574126.486500:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574126.486502:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574126.486505:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574126.486506:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574126.486508:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574126.486511:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574126.486513:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574126.486514:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574126.486516:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574126.486518:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486533:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574126.486536:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486543:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574126.486545:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574126.486549:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574126.486553:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574126.486556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574126.486558:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305730087
00000100:00000001:0.0:1788574126.486562:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305731001
00000100:00000001:0.0:1788574126.486564:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305730087
00000100:00000001:0.0:1788574126.486566:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305731001
00000100:00000001:0.0:1788574126.486569:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574126.486571:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574126.486573:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574126.486577:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574126/real 1788574126]  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574126.486583:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574126.486584:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574126.486585:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574126.486587:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574126.486588:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574126.486589:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574126.486592:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574126.486597:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486601:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574126.486603:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486604:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574126.486605:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574126.486608:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486612:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574126.486614:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486626:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574126.486627:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574126.486629:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486630:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574126.486632:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574126.486633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574126.486635:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574126.486637:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574126.486639:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574126.486645:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393111168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574126.486647:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574126.486649:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486653:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574126.486654:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574126.486655:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574126.486656:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574126.486658:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574126.486660:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393111296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574126.486662:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574126.486664:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574126.486665:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574126.486666:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574126.486668:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574126.486670:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574126.486672:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574126.486673:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574126.486675:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574126.486677:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393111296, portal 4
00000100:00000001:0.0:1788574126.486679:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574126.486680:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:0.0:1788574126.486683:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486688:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574126.486689:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574126.486690:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574126.486692:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393111296, offset 0
00000400:00000200:0.0:1788574126.486694:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574126.486697:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574126.486699:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574126.486701:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574126.486703:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574126.486705:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574126.486706:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574126.486708:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330125:330125:256:4294967295] 192.168.202.110@tcp LPNI seq info [330082:330082:8:4294967295]
00000400:00000200:0.0:1788574126.486711:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574126.486716:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574126.486719:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574126.486721:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574126.486724:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574126.486726:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574126.486728:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574126.486730:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574126.486732:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574126.486734:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574126.486736:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574126.486738:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574126.486740:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574126.486741:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574126.486743:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574126.486745:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486762:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574126.486764:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486768:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574126.486769:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574126.486771:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574126.486773:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574126.486775:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574126.486776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305730233
00000100:00000001:0.0:1788574126.486777:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305731001
00000100:00000001:0.0:1788574126.486779:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305730233
00000100:00000001:0.0:1788574126.486780:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305731001
00000100:00000001:0.0:1788574126.486782:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574126.486783:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574126.486784:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574126.486786:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574126/real 1788574126]  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574126.486791:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574126.486792:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574126.486793:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574126.486795:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574126.486796:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574126.486797:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574126.486799:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574126.486804:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486808:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574126.486809:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486810:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574126.486810:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574126.486813:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486816:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574126.486819:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486823:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574126.486824:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486825:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574126.486826:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486827:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574126.486828:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574126.486829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574126.486831:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574126.486833:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574126.486835:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574126.486844:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393111296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574126.486847:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486848:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574126.486850:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393111168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486854:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574126.486855:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574126.486856:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574126.486857:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574126.486858:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574126.486859:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574126.486861:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574126.486862:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574126.486864:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574126.486865:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574126.486866:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:0.0:1788574126.486867:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574126.486868:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486869:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574126.486871:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393111296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574126.486875:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574126.486876:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574126.486877:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574126.486878:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574126.486879:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574126.486879:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574126.486881:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574126.486881:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788574126.486883:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574126.486883:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574126.486884:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:0.0:1788574126.486886:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574126.486887:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486888:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574126.486891:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574126.486892:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574126.486893:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574126.486894:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574126.869624:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574126.869628:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574126.869631:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574126.997539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574126.997549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574126.997551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574126.997557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574126.997560:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574126.997565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574126.997568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574126.997571:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574126.997575:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574127.445542:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305731001, at 4305731008
00000100:00000040:1.0:1788574127.445542:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305731001, at 4305731008
00000100:00000010:1.0:1788574127.445564:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7060.
00000100:00000010:0.0:1788574127.445572:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788574127.893666:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574127.893670:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574127.893674:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574128.021597:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574128.021609:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574128.021612:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574128.021619:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574128.021624:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574128.021629:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574128.021633:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574128.021636:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574128.021640:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574128.917553:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574128.917558:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574128.917563:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574129.045555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574129.045566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574129.045569:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574129.045575:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574129.045581:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574129.045586:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574129.045589:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574129.045593:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574129.045597:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574129.941575:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574129.941583:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574129.941588:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574130.069524:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574130.069534:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574130.069537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574130.069544:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574130.069548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574130.069553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574130.069557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574130.069561:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574130.069564:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574130.965575:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574130.965580:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574130.965584:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574131.093512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574131.093521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574131.093523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574131.093529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574131.093533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574131.093537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574131.093540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574131.093542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574131.093545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574131.605660:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574131.605666:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574131.605670:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574131.605681:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.605692:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574131.605695:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574131.605701:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574131.605704:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574131.605709:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574131.605715:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393111424:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574131.605722:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574131.605728:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574131.605732:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574131.605737:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574131.605743:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788574131.605747:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574131.605756:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574131.605761:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574131.605768:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574131.605777:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393111424, portal 25
00000100:00000001:0.0:1788574131.605782:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574131.605785:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:0.0:1788574131.605796:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.605813:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574131.605827:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574131.605831:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574131.605837:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393111424, offset 0
00000400:00000200:0.0:1788574131.605847:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574131.605859:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574131.605871:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574131.605883:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574131.605894:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574131.605903:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574131.605912:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574131.605923:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330126:330126:256:4294967295] 192.168.202.110@tcp LPNI seq info [330083:330083:8:4294967295]
00000400:00000200:0.0:1788574131.605954:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574131.605970:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574131.605985:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574131.605992:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574131.606008:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574131.606013:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574131.606019:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574131.606022:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574131.606025:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574131.606028:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574131.606036:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574131.606041:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574131.606046:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574131.606049:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574131.606054:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574131.606059:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606070:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574131.606074:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606082:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574131.606084:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574131.606090:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574131.606093:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574131.606096:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305736121
00000100:00000010:0.0:1788574131.606100:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574131.606106:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574131.606110:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305736121
00000100:00000010:0.0:1788574131.606113:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7f60.
00000100:00000001:0.0:1788574131.606115:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574131.606117:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574131.606119:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574131.606121:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574131.606124:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574131/real 1788574131]  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574131.606132:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574131.606133:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574131.606134:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574131.606136:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574131.606138:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574131.606140:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574131.606142:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574131.606148:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606152:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574131.606154:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606155:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574131.606157:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574131.606160:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606164:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574131.606167:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606172:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574131.606174:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606175:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574131.606176:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606178:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574131.606179:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574131.606181:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574131.606184:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574131.606185:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574131.606188:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574131.606193:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393111424:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574131.606196:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574131.606200:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393111424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606204:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574131.606206:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574131.606208:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788574131.606210:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574131.606211:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574131.606213:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574131.606214:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574131.606216:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788574131.606219:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574131.606221:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574131.606222:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:0.0:1788574131.606224:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574131.606225:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606226:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606232:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574131.606233:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574131.606235:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574131.606236:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574131.606238:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574131.606242:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606249:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574131.606251:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574131.606253:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574131.606255:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574131.606257:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574131.606260:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393111552:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574131.606264:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574131.606266:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574131.606268:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574131.606270:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574131.606273:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788574131.606275:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574131.606279:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574131.606282:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574131.606286:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574131.606290:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393111552, portal 10
00000100:00000001:0.0:1788574131.606291:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574131.606292:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:0.0:1788574131.606297:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606302:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574131.606304:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574131.606305:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574131.606307:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393111552, offset 0
00000400:00000200:0.0:1788574131.606310:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574131.606313:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574131.606315:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574131.606318:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574131.606319:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574131.606321:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574131.606323:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574131.606325:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330127:330127:256:4294967295] 192.168.202.110@tcp LPNI seq info [330084:330084:8:4294967295]
00000400:00000200:0.0:1788574131.606329:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574131.606336:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574131.606341:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574131.606344:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574131.606348:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574131.606351:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574131.606356:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574131.606359:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574131.606364:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574131.606366:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574131.606369:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574131.606373:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574131.606376:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574131.606378:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574131.606380:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574131.606383:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606389:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574131.606422:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606432:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574131.606433:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574131.606436:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574131.606438:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574131.606441:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574131.606443:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305735207
00000100:00000001:0.0:1788574131.606470:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305736121
00000100:00000001:0.0:1788574131.606474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305735207
00000100:00000001:0.0:1788574131.606476:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305736121
00000100:00000001:0.0:1788574131.606478:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574131.606480:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574131.606483:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574131.606488:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574131/real 1788574131]  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574131.606498:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574131.606500:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574131.606502:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574131.606505:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574131.606507:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574131.606510:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574131.606514:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574131.606521:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606526:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574131.606528:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606529:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574131.606530:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574131.606534:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606542:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574131.606547:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606554:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574131.606557:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606559:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574131.606561:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606563:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574131.606565:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574131.606568:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574131.606572:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574131.606575:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574131.606579:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574131.606585:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393111552:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574131.606588:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574131.606590:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606595:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574131.606596:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574131.606597:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574131.606599:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574131.606600:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574131.606602:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393111680:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574131.606604:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574131.606606:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574131.606607:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574131.606608:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574131.606611:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574131.606612:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574131.606616:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574131.606617:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574131.606620:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574131.606622:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393111680, portal 4
00000100:00000001:0.0:1788574131.606623:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574131.606624:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:0.0:1788574131.606628:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606633:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574131.606634:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574131.606636:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574131.606637:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393111680, offset 0
00000400:00000200:0.0:1788574131.606640:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574131.606642:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574131.606645:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574131.606647:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574131.606649:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574131.606651:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574131.606652:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574131.606654:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330128:330128:256:4294967295] 192.168.202.110@tcp LPNI seq info [330085:330085:8:4294967295]
00000400:00000200:0.0:1788574131.606657:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574131.606662:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574131.606665:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574131.606667:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574131.606670:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574131.606672:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574131.606675:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574131.606677:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574131.606679:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574131.606681:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574131.606683:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574131.606686:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574131.606688:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574131.606689:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574131.606691:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574131.606693:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606698:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574131.606700:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606706:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574131.606708:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574131.606711:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574131.606714:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574131.606717:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574131.606719:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305735353
00000100:00000001:0.0:1788574131.606721:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305736121
00000100:00000001:0.0:1788574131.606724:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305735353
00000100:00000001:0.0:1788574131.606726:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305736121
00000100:00000001:0.0:1788574131.606729:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574131.606731:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574131.606733:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574131.606737:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574131/real 1788574131]  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574131.606746:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574131.606747:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574131.606749:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574131.606752:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574131.606753:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574131.606756:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574131.606759:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574131.606767:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606774:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574131.606775:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606777:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574131.606779:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574131.606783:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606790:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574131.606794:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606801:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574131.606804:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606806:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574131.606808:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606809:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574131.606811:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574131.606814:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574131.606817:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574131.606818:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574131.606821:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574131.606827:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393111680:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574131.606829:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606831:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574131.606833:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393111552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606838:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574131.606839:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574131.606841:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788574131.606843:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574131.606844:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574131.606845:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574131.606846:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574131.606847:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788574131.606849:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574131.606850:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574131.606851:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:0.0:1788574131.606853:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574131.606854:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606856:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574131.606858:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393111680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574131.606862:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574131.606862:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574131.606863:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574131.606865:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574131.606866:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574131.606867:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574131.606869:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574131.606871:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788574131.606873:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574131.606874:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574131.606875:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:0.0:1788574131.606878:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574131.606879:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606881:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574131.606885:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574131.606886:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574131.606888:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574131.606890:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574131.989593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574131.989599:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574131.989602:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574132.117501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574132.117511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574132.117515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574132.117523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574132.117530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574132.117536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574132.117542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574132.117545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574132.117549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574132.565489:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305736121, at 4305736128
00000100:00000040:1.0:1788574132.565499:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305736121, at 4305736128
00000100:00000010:0.0:1788574132.565505:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574132.565515:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f60.
00000020:00000001:1.0:1788574133.013593:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574133.013601:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574133.013607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574133.141509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574133.141518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574133.141520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574133.141532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574133.141536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574133.141541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574133.141543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574133.141546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574133.141549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574134.037568:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574134.037572:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574134.037576:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574134.165528:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574134.165536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574134.165538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574134.165544:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574134.165548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574134.165552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574134.165555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574134.165558:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574134.165561:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574135.061595:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574135.061600:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574135.061603:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574135.189549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574135.189561:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574135.189563:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574135.189571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574135.189575:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574135.189581:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574135.189585:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574135.189589:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574135.189593:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574136.085640:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574136.085647:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574136.085653:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574136.725520:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574136.725529:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574136.725532:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574136.725536:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574136.725540:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574136.725546:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11067
00000400:00000200:0.0:1788574136.725555:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574136.725560:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574136.725563:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574136.725565:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725567:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574136.725569:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725571:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574136.725574:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574136.725575:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574136.725577:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574136.725580:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574136.725581:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574136.725583:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574136.725585:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725586:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574136.725587:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725588:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574136.725591:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574136.725594:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725596:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574136.725599:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574136.725601:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574136.725604:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574136.725605:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574136.725607:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574136.725615:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574136.725617:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574136.725620:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574136.725622:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574136.725624:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574136.725627:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574136.725630:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574136.725637:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788574136.725642:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574136.725649:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393111808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574136.725658:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574136.725661:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574136.725664:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393111808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574136.725683:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574136.725686:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574136.725688:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574136.725689:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574136.725691:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574136.725693:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574136.725695:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11067
00000400:00000200:0.0:1788574136.725698:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574136.725700:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574136.725702:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574136.725705:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725707:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574136.725710:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725714:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574136.725717:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574136.725720:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574136.725724:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574136.725725:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574136.725726:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574136.725728:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574136.725730:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725731:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574136.725735:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725738:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574136.725745:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574136.725753:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725760:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574136.725767:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574136.725789:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574136.725793:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574136.725799:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574136.725806:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574136.725814:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574136.725817:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574136.725822:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788574136.725828:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574136.725835:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574136.725839:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574136.725843:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574136.725846:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574136.725851:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574136.725859:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574136.725866:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574136.725878:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393111936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574136.725892:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574136.725896:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574136.725902:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393111936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574136.725914:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574136.725920:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574136.725925:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574136.725929:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574136.725932:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574136.725937:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574136.725943:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11067
00000400:00000200:0.0:1788574136.725953:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574136.725960:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574136.725962:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574136.725966:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725969:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574136.725972:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.725977:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574136.725980:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574136.725982:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574136.725986:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574136.725989:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574136.725992:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574136.725995:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574136.726000:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.726004:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574136.726008:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574136.726011:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574136.726017:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574136.726022:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:2.1:1788574136.726022:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574136.726028:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574136.726029:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
02000000:00000001:0.0:1788574136.726030:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574136.726036:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574136.726040:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574136.726044:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574136.726046:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574136.726050:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574136.726051:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574136.726055:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788574136.726061:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574136.726067:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574136.726071:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574136.726075:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574136.726078:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574136.726081:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574136.726085:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574136.726090:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574136.726098:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393112064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574136.726110:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574136.726111:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574136.726116:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393112064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574136.726127:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574136.726132:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574136.726140:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574136.726145:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11077)
00000800:00000200:2.1:1788574136.726468:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574136.726470:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788574136.726718:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574136.726720:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:0.0:1788574137.109561:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574137.109565:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574137.109570:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574137.685542:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305741241, at 4305741248
00000100:00000040:0.0:1788574137.685548:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305741241, at 4305741248
00000100:00000010:1.0:1788574137.685567:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00000100:00000010:0.0:1788574137.685571:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000800:00000200:1.0:1788574137.749536:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574137.749547:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574137.749554:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574137.749735:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574137.749753:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574137.749789:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574137.749798:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574137.749802:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788574138.133547:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574138.133552:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574138.133556:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574139.157571:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574139.157575:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574139.157577:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574139.285567:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574139.285576:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574139.285579:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574139.285585:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574139.285588:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574139.285592:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574139.285596:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574139.285598:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574139.285601:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574140.181533:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574140.181536:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574140.181539:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574140.309470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574140.309477:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574140.309479:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574140.309483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574140.309485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574140.309488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574140.309491:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574140.309493:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574140.309495:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574141.205562:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574141.205567:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574141.205570:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574141.333474:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574141.333481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574141.333484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574141.333490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574141.333494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574141.333498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574141.333500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574141.333503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574141.333506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574141.845568:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574141.845581:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574141.845584:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574141.845589:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574141.845595:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574141.845603:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11072
00000400:00000200:1.0:1788574141.845612:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574141.845633:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574141.845637:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574141.845640:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845643:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574141.845646:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845648:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574141.845651:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574141.845653:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574141.845655:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574141.845658:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574141.845660:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574141.845661:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574141.845664:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845666:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574141.845667:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845669:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574141.845671:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574141.845674:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845676:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574141.845678:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574141.845681:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574141.845683:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574141.845685:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574141.845687:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574141.845694:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788574141.845697:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574141.845699:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574141.845701:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574141.845703:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574141.845705:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574141.845708:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574141.845715:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574141.845720:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574141.845727:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393112192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574141.845735:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574141.845738:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574141.845742:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393112192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574141.845757:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574141.845760:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574141.845763:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574141.845764:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574141.845765:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574141.845767:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574141.845769:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11072
00000400:00000200:1.0:1788574141.845773:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574141.845776:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574141.845778:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574141.845780:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845782:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574141.845784:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845786:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574141.845787:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574141.845788:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574141.845790:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574141.845792:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574141.845795:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574141.845797:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574141.845802:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845804:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574141.845808:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845810:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574141.845814:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574141.845819:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845824:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574141.845828:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574141.845835:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574141.845839:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574141.845843:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574141.845846:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574141.845852:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574141.845856:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574141.845861:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788574141.845865:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574141.845868:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574141.845872:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574141.845876:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574141.845879:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574141.845882:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574141.845886:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788574141.845891:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574141.845898:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393112320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574141.845908:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574141.845911:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574141.845917:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393112320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574141.845932:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574141.845936:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574141.845944:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574141.845946:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574141.845949:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574141.845953:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574141.845961:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11072
00000400:00000200:1.0:1788574141.845968:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574141.845973:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574141.845976:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574141.845981:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845986:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574141.845988:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.845990:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574141.845995:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574141.845998:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574141.845999:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574141.846000:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574141.846004:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574141.846008:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574141.846014:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.846017:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574141.846022:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574141.846026:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574141.846031:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574141.846037:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574141.846040:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574141.846044:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574141.846048:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574141.846051:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574141.846052:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574141.846054:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574141.846058:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574141.846062:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574141.846068:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788574141.846073:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574141.846078:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574141.846079:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574141.846081:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574141.846083:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574141.846086:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574141.846090:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788574141.846094:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574141.846102:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393112448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1788574141.846106:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574141.846111:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574141.846113:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574141.846116:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393112448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788574141.846122:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:1.0:1788574141.846125:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574141.846130:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574141.846133:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574141.846140:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11083)
00000800:00000200:2.1:1788574141.846509:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574141.846512:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788574141.846794:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574141.846796:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:1.0:1788574142.229553:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574142.229556:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574142.229560:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574142.357492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574142.357498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574142.357499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574142.357503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574142.357506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574142.357510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574142.357512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574142.357514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574142.357516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574142.805491:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305746361, at 4305746368
00000100:00000040:1.0:1788574142.805491:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305746361, at 4305746368
00000100:00000010:0.0:1788574142.805507:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000100:00000010:1.0:1788574142.805509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000020:00000001:0.0:1788574143.253553:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574143.253556:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574143.253558:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574143.381453:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574143.381459:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574143.381460:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574143.381464:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574143.381467:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574143.381471:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574143.381473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574143.381475:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574143.381478:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574144.277574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574144.277578:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574144.277581:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574144.405509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574144.405517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574144.405529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574144.405549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574144.405554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574144.405558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574144.405561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574144.405564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574144.405568:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574145.301628:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574145.301634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574145.301637:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574145.429576:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574145.429586:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574145.429589:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574145.429597:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574145.429602:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574145.429607:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574145.429610:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574145.429614:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574145.429618:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574146.325568:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574146.325573:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574146.325577:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788574146.965483:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574146.965494:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574146.965497:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574146.965501:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574146.965507:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574146.965514:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11078
00000400:00000200:1.0:1788574146.965523:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574146.965528:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574146.965531:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574146.965534:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965536:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574146.965538:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965540:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574146.965543:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574146.965545:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574146.965547:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574146.965549:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574146.965551:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574146.965553:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574146.965555:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965557:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574146.965558:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965559:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574146.965562:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574146.965564:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965566:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574146.965568:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574146.965570:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574146.965572:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574146.965574:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574146.965575:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574146.965582:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788574146.965585:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574146.965588:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574146.965589:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574146.965591:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574146.965593:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574146.965596:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574146.965603:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574146.965607:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574146.965614:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393112576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574146.965622:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574146.965625:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574146.965628:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393112576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574146.965647:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574146.965649:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574146.965652:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574146.965653:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574146.965654:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574146.965657:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574146.965658:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11078
00000400:00000200:1.0:1788574146.965661:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574146.965663:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574146.965665:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574146.965666:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965668:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574146.965670:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965671:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574146.965673:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574146.965674:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574146.965675:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574146.965677:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574146.965678:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574146.965680:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574146.965682:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965683:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574146.965685:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965687:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574146.965688:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574146.965691:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965695:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574146.965697:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574146.965702:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574146.965704:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574146.965707:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574146.965710:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574146.965713:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574146.965715:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574146.965721:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788574146.965725:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574146.965729:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574146.965731:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574146.965735:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574146.965737:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574146.965743:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574146.965748:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574146.965751:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574146.965757:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393112704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574146.965768:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574146.965773:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574146.965779:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393112704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574146.965788:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574146.965790:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574146.965794:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574146.965796:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574146.965798:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574146.965801:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574146.965806:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11078
00000400:00000200:1.0:1788574146.965808:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574146.965811:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574146.965814:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574146.965818:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965821:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574146.965822:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965824:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574146.965828:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574146.965830:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574146.965834:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574146.965836:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574146.965838:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574146.965843:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574146.965846:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965847:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574146.965850:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965852:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574146.965855:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574146.965857:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574146.965859:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574146.965861:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574146.965864:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574146.965866:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574146.965871:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574146.965873:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574146.965877:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574146.965878:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574146.965881:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788574146.965884:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574146.965887:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574146.965889:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574146.965893:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574146.965895:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574146.965896:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574146.965899:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788574146.965901:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574146.965904:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393112832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574146.965921:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574146.965925:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574146.965928:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393112832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574146.965938:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574146.965941:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574146.965944:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574146.965950:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11088)
00000800:00000200:2.1:1788574146.966046:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574146.966050:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574146.966582:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574146.966585:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574146.966920:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574146.966923:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:1.0:1788574147.349602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574147.349606:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574147.349609:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574147.925488:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305751481, at 4305751488
00000100:00000040:1.0:1788574147.925488:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305751481, at 4305751488
00000100:00000010:0.0:1788574147.925503:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000100:00000010:1.0:1788574147.925504:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00000020:00000001:1.0:1788574148.373574:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574148.373579:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574148.373583:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574149.397620:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574149.397624:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574149.397627:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574150.421627:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574150.421631:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574150.421634:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574151.445593:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574151.445597:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574151.445600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788574152.085524:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574152.085535:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574152.085538:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574152.085543:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574152.085548:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574152.085557:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11083
00000400:00000200:1.0:1788574152.085567:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574152.085574:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574152.085578:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574152.085581:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085584:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574152.085587:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085589:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574152.085594:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574152.085596:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574152.085598:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574152.085601:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574152.085603:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574152.085605:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574152.085607:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085609:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574152.085610:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085612:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574152.085614:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574152.085617:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085619:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574152.085622:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574152.085625:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574152.085627:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574152.085629:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574152.085632:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574152.085643:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788574152.085646:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574152.085649:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574152.085650:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574152.085653:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574152.085656:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574152.085661:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574152.085670:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788574152.085677:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574152.085686:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393112960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574152.085695:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574152.085700:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574152.085704:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393112960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574152.085721:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574152.085724:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574152.085726:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574152.085728:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574152.085729:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574152.085732:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574152.085734:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11083
00000400:00000200:1.0:1788574152.085738:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574152.085741:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574152.085743:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574152.085749:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085751:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574152.085755:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085759:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574152.085761:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574152.085763:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574152.085765:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574152.085767:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574152.085773:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574152.085779:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574152.085786:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085792:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574152.085798:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085805:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574152.085813:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574152.085821:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574152.085832:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574152.085836:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574152.085845:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574152.085848:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574152.085850:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574152.085857:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574152.085863:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574152.085867:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574152.085873:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788574152.085878:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574152.085882:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574152.085884:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574152.085890:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574152.085896:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574152.085900:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574152.085908:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574152.085914:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574152.085924:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393113088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574152.085935:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574152.085938:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574152.085943:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393113088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574152.085958:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574152.085961:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574152.085964:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574152.085966:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574152.085969:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574152.085971:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574152.085976:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11083
00000400:00000200:1.0:1788574152.085982:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574152.085989:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574152.085994:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574152.086001:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.086005:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574152.086011:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.086015:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574152.086018:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574152.086020:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574152.086023:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574152.086027:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574152.086033:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574152.086034:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574152.086037:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.086042:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574152.086048:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574152.086052:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574152.086057:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574152.086063:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574152.086066:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574152.086067:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574152.086070:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574152.086073:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574152.086074:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574152.086076:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574152.086080:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574152.086085:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574152.086089:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788574152.086094:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.1:1788574152.086098:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1788574152.086099:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574152.086102:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574152.086105:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1788574152.086106:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574152.086109:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574152.086112:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574152.086118:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788574152.086122:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574152.086127:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393113216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574152.086139:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574152.086143:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574152.086150:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393113216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574152.086162:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574152.086165:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574152.086169:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574152.086174:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11093)
00000800:00000200:2.1:1788574152.086552:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574152.086554:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574152.086751:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574152.086752:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1788574152.469585:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574152.469591:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574152.469595:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574153.045503:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305756601, at 4305756608
00000100:00000040:0.0:1788574153.045507:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305756601, at 4305756608
00000100:00000010:1.0:1788574153.045524:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000100:00000010:0.0:1788574153.045531:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000020:00000001:1.0:1788574153.493588:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574153.493591:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574153.493594:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574154.517557:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574154.517562:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574154.517566:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574155.541556:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574155.541560:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574155.541563:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574156.565579:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574156.565584:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574156.565588:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574157.206033:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574157.206041:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574157.206427:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574157.206429:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574157.206623:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574157.206624:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788574157.589537:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574157.589541:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574157.589544:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574158.165450:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305761721, at 4305761728
00000100:00000040:1.0:1788574158.165458:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305761721, at 4305761728
00000100:00000010:0.0:1788574158.165466:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00000100:00000010:1.0:1788574158.165467:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000020:00000001:0.0:1788574158.613559:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574158.613563:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574158.613566:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574159.637565:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574159.637570:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574159.637574:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574160.661573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574160.661577:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574160.661581:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574161.685545:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574161.685550:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574161.685554:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574162.326079:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574162.326082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788574162.326545:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574162.326549:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788574162.326863:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574162.326866:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:1.0:1788574162.709602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574162.709606:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574162.709608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574163.285458:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305766841, at 4305766848
00000100:00000010:1.0:1788574163.285472:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00000100:00000040:0.0:1788574163.285476:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305766841, at 4305766848
00000100:00000010:0.0:1788574163.285488:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000020:00000001:1.0:1788574163.733555:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574163.733559:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574163.733562:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574164.757541:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574164.757545:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574164.757547:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574165.781580:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574165.781583:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574165.781585:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574166.805557:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574166.805561:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574166.805565:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574167.446041:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574167.446047:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574167.446441:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574167.446443:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574167.446643:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574167.446645:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:0.0:1788574167.829539:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574167.829542:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574167.829545:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574168.405506:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305771961, at 4305771968
00000100:00000040:1.0:1788574168.405506:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305771961, at 4305771968
00000100:00000010:0.0:1788574168.405531:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000100:00000010:1.0:1788574168.405534:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000020:00000001:1.0:1788574168.853617:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574168.853621:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574168.853624:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574169.877538:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574169.877542:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574169.877546:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574170.901538:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574170.901542:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574170.901545:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574171.925639:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574171.925643:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574171.925646:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574172.565586:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574172.565599:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574172.565602:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574172.565607:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574172.565612:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574172.565625:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11103
00000400:00000200:0.0:1788574172.565635:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574172.565641:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574172.565644:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574172.565647:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565650:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574172.565652:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565654:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574172.565658:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574172.565660:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574172.565663:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574172.565666:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574172.565668:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574172.565670:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574172.565673:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565674:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574172.565675:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565676:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574172.565679:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574172.565685:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565687:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574172.565690:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574172.565693:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574172.565696:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574172.565697:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574172.565700:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574172.565711:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788574172.565713:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574172.565717:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574172.565719:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574172.565722:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574172.565725:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574172.565729:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574172.565737:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574172.565744:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574172.565754:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393114496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574172.565763:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574172.565770:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574172.565776:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393114496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574172.565812:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574172.565818:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574172.565821:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574172.565822:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574172.565823:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574172.565826:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574172.565828:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11103
00000400:00000200:0.0:1788574172.565831:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574172.565834:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574172.565836:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574172.565839:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565841:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574172.565843:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565844:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574172.565846:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574172.565848:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574172.565849:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574172.565851:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574172.565852:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574172.565854:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574172.565857:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565859:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574172.565860:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565862:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574172.565864:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574172.565867:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574172.565870:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574172.565874:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574172.565881:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574172.565888:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574172.565891:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574172.565896:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574172.565899:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574172.565902:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574172.565906:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574172.565909:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574172.565914:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574172.565916:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574172.565921:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574172.565926:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574172.565931:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574172.565938:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574172.565945:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574172.565954:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393114624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574172.565968:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574172.565972:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574172.565988:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393114624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574172.566013:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574172.566017:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574172.566023:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574172.566027:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574172.566031:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574172.566034:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574172.566041:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11103
00000400:00000200:0.0:1788574172.566044:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574172.566046:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574172.566049:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574172.566055:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.566056:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574172.566058:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.566059:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574172.566061:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574172.566065:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574172.566067:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574172.566071:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574172.566073:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574172.566076:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574172.566084:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.566086:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574172.566088:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574172.566100:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574172.566103:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574172.566107:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574172.566112:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574172.566114:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574172.566117:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574172.566124:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574172.566126:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574172.566130:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574172.566133:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574172.566136:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574172.566143:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574172.566146:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574172.566150:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574172.566152:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574172.566156:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574172.566158:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574172.566162:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574172.566167:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788574172.566174:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574172.566185:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393114752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574172.566208:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574172.566210:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574172.566219:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393114752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574172.566235:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574172.566242:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574172.566253:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574172.566259:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11113)
00000800:00000200:2.1:1788574172.566324:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574172.566327:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574172.566820:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574172.566822:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574172.567077:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574172.567080:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:1.0:1788574172.949610:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574172.949617:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574172.949622:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574173.525463:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305777082, at 4305777088
00000100:00000010:0.0:1788574173.525479:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000100:00000040:1.0:1788574173.525507:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305777082, at 4305777088
00000100:00000010:1.0:1788574173.525521:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000020:00000001:0.0:1788574173.973534:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574173.973538:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574173.973541:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574174.997590:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574174.997596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574174.997601:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574175.125528:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574175.125537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574175.125539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574175.125545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574175.125549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574175.125552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574175.125554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574175.125556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574175.125559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788574175.637650:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788574175.637655:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574175.637665:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788574175.637669:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574175.637670:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788574176.021689:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574176.021696:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574176.021701:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574176.149491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574176.149498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574176.149500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574176.149504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574176.149507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574176.149510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574176.149512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574176.149514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574176.149516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574177.045537:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574177.045541:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574177.045543:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574177.173461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574177.173468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574177.173470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574177.173474:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574177.173477:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574177.173480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574177.173482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574177.173484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574177.173486:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574177.685502:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574177.685510:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574177.685512:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574177.685515:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574177.685520:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574177.685525:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11108
00000400:00000200:1.0:1788574177.685533:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574177.685539:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574177.685541:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574177.685544:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685546:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574177.685548:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685550:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574177.685553:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574177.685554:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574177.685556:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574177.685559:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574177.685561:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574177.685563:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574177.685567:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685569:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574177.685570:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685573:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574177.685576:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574177.685580:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685583:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574177.685586:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574177.685589:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574177.685593:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574177.685595:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574177.685598:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574177.685608:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788574177.685612:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574177.685615:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574177.685616:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574177.685619:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574177.685622:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574177.685626:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574177.685632:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574177.685638:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574177.685646:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393114880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574177.685654:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574177.685657:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574177.685661:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393114880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574177.685679:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574177.685681:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574177.685684:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574177.685685:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574177.685687:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574177.685689:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574177.685692:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11108
00000400:00000200:1.0:1788574177.685696:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574177.685700:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574177.685702:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574177.685706:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685708:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574177.685713:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685717:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574177.685719:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574177.685721:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574177.685723:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574177.685726:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574177.685730:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574177.685734:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574177.685742:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685746:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574177.685752:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685757:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574177.685763:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574177.685771:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685777:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574177.685782:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574177.685791:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574177.685795:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574177.685801:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574177.685807:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574177.685812:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574177.685816:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574177.685821:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788574177.685824:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574177.685829:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574177.685831:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574177.685832:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574177.685833:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574177.685835:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574177.685842:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788574177.685847:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574177.685853:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393115008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574177.685870:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574177.685874:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574177.685883:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393115008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574177.685899:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574177.685903:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574177.685908:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574177.685910:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574177.685913:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574177.685917:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574177.685922:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11108
00000400:00000200:1.0:1788574177.685930:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574177.685935:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574177.685936:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574177.685942:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685946:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574177.685950:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.685957:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574177.685960:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574177.685964:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574177.685967:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574177.685971:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574177.685978:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574177.685985:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574177.685996:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.686001:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574177.686007:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574177.686014:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574177.686020:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574177.686027:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574177.686032:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574177.686034:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574177.686037:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574177.686042:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574177.686044:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574177.686046:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574177.686051:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574177.686052:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574177.686058:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000800:00000200:2.1:1788574177.686060:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:1.0:1788574177.686062:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574177.686066:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574177.686067:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574177.686068:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574177.686069:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574177.686071:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574177.686074:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788574177.686076:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574177.686082:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393115136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574177.686096:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574177.686099:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574177.686104:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393115136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574177.686112:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574177.686116:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574177.686120:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574177.686125:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11118)
00000800:00000200:2.1:1788574177.686299:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574177.686467:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788574178.069573:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574178.069577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574178.069581:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574178.197450:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574178.197457:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574178.197459:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574178.197463:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574178.197466:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574178.197469:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574178.197471:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574178.197473:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574178.197476:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574178.645462:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305782201, at 4305782208
00000100:00000040:0.0:1788574178.645464:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305782201, at 4305782208
00000100:00000010:0.0:1788574178.645479:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000100:00000010:1.0:1788574178.645480:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000020:00000001:1.0:1788574179.093632:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574179.093637:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574179.093641:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574179.221478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574179.221484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574179.221486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574179.221490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574179.221493:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574179.221495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574179.221497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574179.221499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574179.221501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574180.117528:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574180.117534:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574180.117537:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574180.245470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574180.245476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574180.245477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574180.245482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574180.245484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574180.245487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574180.245490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574180.245492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574180.245494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1788574180.757646:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:0.0:1788574180.757683:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1788574180.757692:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574180.757695:0:2350:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:0.0:1788574180.757699:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574180.757703:0:2350:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1788574180.757706:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788574180.757709:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:0.0:1788574180.757714:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788574180.757718:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788574180.757724:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65ee0
00000400:00000010:0.0:1788574180.757726:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65ee0.
00000100:00000001:0.0:1788574180.757730:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574180.757737:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f7800 x1875444393114880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574180.757758:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574180.757762:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393114880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574180.757771:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574180.757773:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788574180.757784:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788574180.757787:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133800.
00000400:00000200:0.0:1788574180.757791:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788574180.757796:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788574180.757799:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:0.0:1788574180.757801:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:0.0:1788574180.757805:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574180.757809:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f6d80 x1875444393115008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574180.757821:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574180.757828:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393115008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574180.757840:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574180.757845:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788574180.757872:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788574180.757877:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:0.0:1788574180.757884:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788574180.757899:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788574180.757909:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65dd0
00000400:00000010:0.0:1788574180.757915:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65dd0.
00000100:00000001:0.0:1788574180.757921:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574180.757931:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f7480 x1875444393115136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574180.757942:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574180.757946:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393115136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574180.757959:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574180.757965:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000001:1.0:1788574181.141558:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574181.141563:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574181.141566:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574181.269508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574181.269515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574181.269517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574181.269522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574181.269526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574181.269530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574181.269533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574181.269536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574181.269539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574182.165630:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574182.165634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574182.165638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574182.293465:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574182.293471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574182.293473:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574182.293477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574182.293481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574182.293484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574182.293486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574182.293488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574182.293490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574182.805496:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574182.805508:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574182.805511:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574182.805515:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574182.805520:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574182.805528:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11113
00000400:00000200:1.0:1788574182.805539:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574182.805546:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574182.805550:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574182.805555:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805559:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574182.805561:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805563:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574182.805566:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574182.805567:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574182.805570:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574182.805572:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574182.805574:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574182.805575:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574182.805577:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805579:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574182.805581:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805582:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574182.805585:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574182.805587:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805589:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574182.805591:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574182.805594:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574182.805596:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574182.805597:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574182.805599:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574182.805604:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788574182.805607:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574182.805610:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574182.805612:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574182.805615:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574182.805618:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574182.805622:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574182.805629:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574182.805636:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574182.805645:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393115264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574182.805655:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574182.805659:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574182.805664:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393115264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574182.805686:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574182.805690:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574182.805693:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574182.805695:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574182.805697:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574182.805700:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574182.805703:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11113
00000400:00000200:1.0:1788574182.805708:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574182.805713:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574182.805715:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574182.805719:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805722:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574182.805729:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805734:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574182.805739:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574182.805742:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574182.805745:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574182.805749:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574182.805754:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574182.805760:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574182.805769:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805776:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574182.805781:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805786:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574182.805791:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574182.805800:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574182.805807:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574182.805812:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574182.805819:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574182.805825:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574182.805829:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574182.805836:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574182.805841:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574182.805845:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574182.805852:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788574182.805857:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574182.805860:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574182.805862:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574182.805868:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574182.805869:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574182.805871:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574182.805877:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574182.805884:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574182.805894:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393115392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574182.805917:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574182.805922:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574182.805932:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393115392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574182.805949:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574182.805954:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574182.805972:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574182.805975:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574182.805977:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574182.805980:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574182.805985:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11113
00000400:00000200:1.0:1788574182.805994:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574182.806000:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574182.806005:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574182.806010:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.806014:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574182.806020:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.806024:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574182.806028:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574182.806032:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574182.806035:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574182.806040:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574182.806045:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574182.806051:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574182.806058:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.806061:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574182.806066:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574182.806071:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574182.806075:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574182.806082:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574182.806086:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574182.806089:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574182.806093:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574182.806097:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574182.806099:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574182.806101:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574182.806106:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574182.806108:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574182.806115:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788574182.806120:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574182.806125:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574182.806128:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574182.806131:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574182.806132:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574182.806135:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574182.806139:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574182.806139:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
02000000:00000010:1.0:1788574182.806146:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788574182.806153:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574182.806162:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393115520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574182.806177:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574182.806182:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574182.806188:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393115520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574182.806203:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574182.806209:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574182.806220:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574182.806229:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11124)
00000800:00000200:2.1:1788574182.806700:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574182.806703:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788574182.807058:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574182.807061:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1788574183.189552:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574183.189555:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574183.189559:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574183.317517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574183.317523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574183.317525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574183.317528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574183.317531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574183.317534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574183.317536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574183.317538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574183.317541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574183.765489:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305787321, at 4305787328
00000100:00000040:1.0:1788574183.765492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305787321, at 4305787328
00000100:00000010:0.0:1788574183.765504:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000100:00000010:1.0:1788574183.765507:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000020:00000001:1.0:1788574184.213686:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574184.213692:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574184.213696:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574184.341477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574184.341484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574184.341486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574184.341489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574184.341492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574184.341495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574184.341498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574184.341500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574184.341502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574185.237566:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574185.237570:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574185.237572:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574185.365500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574185.365507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574185.365509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574185.365514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574185.365517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574185.365520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574185.365524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574185.365526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574185.365529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574186.261541:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574186.261544:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574186.261547:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574186.389472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574186.389478:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574186.389480:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574186.389484:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574186.389486:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574186.389489:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574186.389491:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574186.389493:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574186.389495:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574187.285543:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574187.285546:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574187.285549:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574187.413505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574187.413514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574187.413516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574187.413523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574187.413527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574187.413531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574187.413534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574187.413537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574187.413541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574187.926072:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574187.926081:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788574187.926587:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574187.926590:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788574187.926848:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574187.926850:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788574188.437479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574188.437485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574188.437486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574188.437491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574188.437493:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574188.437496:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574188.437498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574188.437500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574188.437502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574188.885472:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305792441, at 4305792448
00000100:00000040:1.0:1788574188.885473:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305792441, at 4305792448
00000100:00000010:0.0:1788574188.885488:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000100:00000010:1.0:1788574188.885490:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000020:00000001:0.0:1788574189.333638:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574189.333641:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574189.333644:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574190.357612:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574190.357616:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574190.357619:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574192.405618:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574192.405622:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574192.405625:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574193.045978:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574193.045983:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788574193.046342:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574193.046344:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788574193.046645:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574193.046647:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:1.0:1788574193.429586:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574193.429592:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574193.429596:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574194.005550:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305797561, at 4305797568
00000100:00000040:0.0:1788574194.005557:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305797561, at 4305797568
00000100:00000010:1.0:1788574194.005570:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00000100:00000010:0.0:1788574194.005575:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000020:00000001:0.0:1788574195.477551:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574195.477555:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574195.477558:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574196.501556:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574196.501560:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574196.501563:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788574198.165576:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574198.165588:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574198.165591:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574198.165596:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574198.165601:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574198.165609:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11129
00000400:00000200:1.0:1788574198.165619:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574198.165626:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574198.165628:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574198.165632:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165635:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574198.165638:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165641:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574198.165646:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574198.165649:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574198.165653:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574198.165657:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574198.165659:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574198.165663:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574198.165666:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165669:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574198.165671:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165674:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574198.165677:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574198.165681:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165683:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574198.165686:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574198.165689:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574198.165692:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574198.165694:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574198.165697:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574198.165715:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788574198.165718:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574198.165723:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574198.165724:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574198.165728:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574198.165731:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574198.165736:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574198.165747:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574198.165754:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574198.165764:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393116416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574198.165774:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574198.165778:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574198.165782:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393116416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574198.165799:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574198.165801:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574198.165804:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574198.165806:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574198.165807:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574198.165810:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574198.165812:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11129
00000400:00000200:1.0:1788574198.165816:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574198.165819:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574198.165821:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574198.165825:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165827:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574198.165831:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165833:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574198.165837:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574198.165840:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574198.165842:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574198.165844:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574198.165847:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574198.165849:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574198.165855:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165859:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574198.165863:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165867:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574198.165870:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574198.165875:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574198.165879:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574198.165881:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574198.165890:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574198.165894:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574198.165899:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574198.165904:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574198.165907:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574198.165910:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574198.165914:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788574198.165917:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574198.165920:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574198.165922:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574198.165923:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574198.165928:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574198.165930:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574198.165938:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788574198.165943:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574198.165949:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393116544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574198.165957:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574198.165959:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574198.165963:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393116544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574198.165974:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574198.165978:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574198.165983:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574198.165986:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574198.165989:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574198.165996:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574198.166004:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11129
00000400:00000200:1.0:1788574198.166015:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574198.166023:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574198.166029:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574198.166035:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.166039:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574198.166045:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.166048:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574198.166051:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574198.166055:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574198.166057:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574198.166059:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574198.166062:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574198.166066:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574198.166087:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.166091:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574198.166096:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574198.166099:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574198.166102:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574198.166108:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574198.166113:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574198.166118:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574198.166122:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574198.166125:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574198.166127:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574198.166129:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574198.166134:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574198.166136:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574198.166143:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788574198.166148:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574198.166155:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574198.166157:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574198.166159:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574198.166162:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574198.166164:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574198.166168:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788574198.166174:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574198.166181:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393116672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1788574198.166186:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574198.166193:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:1.0:1788574198.166197:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574198.166201:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574198.166206:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393116672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574198.166216:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574198.166220:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574198.166223:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574198.166229:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11139)
00000800:00000200:2.1:1788574198.166618:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574198.166620:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788574198.166878:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574198.166880:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:0.0:1788574198.549554:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574198.549558:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574198.549562:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574199.125491:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305802681, at 4305802688
00000100:00000040:1.0:1788574199.125505:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305802681, at 4305802688
00000100:00000010:0.0:1788574199.125508:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00000100:00000010:1.0:1788574199.125523:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000020:00000001:1.0:1788574199.573651:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574199.573657:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574199.573661:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574201.621547:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574201.621550:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574201.621553:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574202.645527:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574202.645531:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574202.645534:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788574203.285485:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574203.285494:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574203.285497:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574203.285500:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574203.285504:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574203.285511:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11134
00000400:00000200:1.0:1788574203.285520:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574203.285526:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574203.285528:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574203.285531:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285534:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574203.285537:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285539:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574203.285542:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574203.285543:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574203.285545:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574203.285548:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574203.285549:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574203.285551:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574203.285554:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285556:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574203.285557:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285558:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574203.285561:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574203.285563:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285565:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574203.285567:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574203.285569:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574203.285572:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574203.285573:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574203.285575:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574203.285583:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788574203.285585:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574203.285588:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574203.285589:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574203.285591:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574203.285594:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574203.285597:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574203.285604:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574203.285615:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574203.285624:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393116800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574203.285632:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574203.285635:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574203.285638:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393116800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574203.285654:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574203.285657:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574203.285659:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574203.285660:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574203.285662:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574203.285664:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574203.285666:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11134
00000400:00000200:1.0:1788574203.285669:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574203.285671:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574203.285673:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574203.285676:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285677:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574203.285679:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285681:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574203.285683:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574203.285684:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574203.285686:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574203.285687:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574203.285690:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574203.285695:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574203.285698:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285701:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574203.285707:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285710:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574203.285714:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574203.285719:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285724:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574203.285727:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574203.285734:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574203.285739:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574203.285743:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574203.285748:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574203.285754:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574203.285757:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574203.285763:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788574203.285767:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574203.285770:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574203.285782:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574203.285784:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574203.285786:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574203.285788:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574203.285794:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574203.285801:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574203.285809:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393116928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574203.285818:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574203.285820:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574203.285825:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393116928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574203.285839:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574203.285844:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574203.285852:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574203.285856:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574203.285858:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574203.285865:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574203.285870:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11134
00000400:00000200:1.0:1788574203.285877:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574203.285883:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574203.285887:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574203.285895:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285900:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574203.285904:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285917:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574203.285926:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574203.285930:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574203.285933:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574203.285937:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574203.285942:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574203.285945:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574203.285953:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285956:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574203.285958:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285960:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574203.285963:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574203.285968:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574203.285972:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574203.285974:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574203.285978:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574203.285981:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574203.285984:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574203.285987:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574203.285991:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574203.285993:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574203.285997:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788574203.286002:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574203.286004:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574203.286005:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574203.286009:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574203.286011:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574203.286016:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574203.286022:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000800:00000200:2.1:1788574203.286024:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574203.286026:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574203.286027:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00100000:1.0:1788574203.286035:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393117056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574203.286046:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574203.286048:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574203.286056:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393117056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574203.286066:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574203.286069:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574203.286074:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574203.286081:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11144)
00000800:00000200:2.1:1788574203.286448:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574203.286449:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788574203.286670:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574203.286672:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:0.0:1788574204.245487:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305807801, at 4305807808
00000100:00000040:1.0:1788574204.245487:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305807801, at 4305807808
00000100:00000010:0.0:1788574204.245501:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00000100:00000010:1.0:1788574204.245504:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000020:00000001:0.0:1788574204.693556:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574204.693559:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574204.693561:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574205.717589:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574205.717593:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574205.717595:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574207.765601:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574207.765613:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574207.765616:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574208.406087:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574208.406091:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788574208.406459:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574208.406461:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788574208.406725:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574208.406727:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:1.0:1788574208.789640:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574208.789653:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574208.789656:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574209.365493:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305812921, at 4305812928
00000100:00000040:1.0:1788574209.365493:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305812921, at 4305812928
00000100:00000010:1.0:1788574209.365508:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000100:00000010:0.0:1788574209.365511:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000020:00000001:0.0:1788574210.837555:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574210.837558:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574210.837560:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574210.965516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574210.965523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574210.965525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574210.965529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574210.965532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574210.965535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574210.965538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574210.965540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574210.965542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574211.861565:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574211.861568:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574211.861571:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574211.989485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574211.989493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574211.989495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574211.989501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574211.989505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574211.989509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574211.989512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574211.989515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574211.989518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574213.013490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574213.013496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574213.013498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574213.013502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574213.013504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574213.013508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574213.013510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574213.013512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574213.013514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574213.526056:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574213.526062:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788574213.526474:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574213.526476:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788574213.526693:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574213.526695:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:0.0:1788574213.909674:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574213.909679:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574213.909683:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574214.037461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574214.037467:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574214.037469:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574214.037473:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574214.037475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574214.037478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574214.037480:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574214.037481:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574214.037483:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574214.485474:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305818041, at 4305818048
00000100:00000040:1.0:1788574214.485474:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305818041, at 4305818048
00000100:00000010:0.0:1788574214.485489:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000100:00000010:1.0:1788574214.485491:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000800:00000200:1.0:1788574214.549508:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574214.549515:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574214.549520:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574214.549656:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574214.549667:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574214.549669:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574214.549675:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574214.549678:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788574214.933542:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574214.933546:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574214.933549:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574215.061476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574215.061483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574215.061485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574215.061489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574215.061492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574215.061495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574215.061497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574215.061499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574215.061502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574216.085511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574216.085520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574216.085522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574216.085528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574216.085532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574216.085536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574216.085540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574216.085542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574216.085546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574216.981585:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574216.981588:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574216.981591:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574217.109490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574217.109496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574217.109498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574217.109502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574217.109505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574217.109508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574217.109510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574217.109512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574217.109515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574218.005554:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574218.005559:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574218.005563:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574218.133459:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574218.133465:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574218.133468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574218.133473:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574218.133476:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574218.133479:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574218.133481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574218.133483:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574218.133485:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574218.645710:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574218.645716:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574218.645719:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574218.645729:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.645748:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574218.645751:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574218.645756:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574218.645761:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574218.645765:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574218.645771:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393117952:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574218.645778:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574218.645783:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574218.645786:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574218.645789:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574218.645794:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574218.645799:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574218.645814:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574218.645822:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574218.645831:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574218.645839:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393117952, portal 25
00000100:00000001:0.0:1788574218.645844:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574218.645847:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:0.0:1788574218.645857:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.645870:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574218.645877:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574218.645882:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574218.645896:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393117952, offset 0
00000400:00000200:0.0:1788574218.645908:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574218.645918:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574218.645927:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574218.645935:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574218.645942:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574218.645950:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574218.645956:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574218.645968:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330177:330177:256:4294967295] 192.168.202.110@tcp LPNI seq info [330134:330134:8:4294967295]
00000400:00000200:0.0:1788574218.645978:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574218.645994:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574218.646004:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574218.646011:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574218.646024:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574218.646030:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1788574218.646040:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574218.646046:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:0.0:1788574218.646055:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574218.646058:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574218.646064:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574218.646073:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574218.646081:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574218.646085:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574218.646092:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574218.646099:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646118:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574218.646124:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646133:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574218.646136:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574218.646143:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574218.646147:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574218.646151:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305823161
00000100:00000010:0.0:1788574218.646157:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574218.646164:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574218.646170:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305823161
00000100:00000010:0.0:1788574218.646173:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7240.
00000100:00000001:0.0:1788574218.646177:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574218.646180:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574218.646182:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574218.646185:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574218.646191:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574218/real 1788574218]  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574218.646200:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574218.646202:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574218.646205:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574218.646208:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574218.646210:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574218.646212:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574218.646216:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574218.646225:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646230:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574218.646231:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646233:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574218.646235:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574218.646238:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646243:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574218.646246:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646251:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574218.646253:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646254:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574218.646254:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646256:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574218.646258:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574218.646260:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574218.646263:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574218.646264:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574218.646267:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574218.646273:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393117952:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574218.646276:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646277:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574218.646279:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393117952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646283:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574218.646285:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574218.646287:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574218.646289:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574218.646291:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574218.646292:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574218.646294:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574218.646296:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788574218.646299:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574218.646300:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574218.646301:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:0.0:1788574218.646303:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574218.646304:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646305:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646311:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574218.646312:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574218.646314:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574218.646315:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574218.646316:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574218.646318:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646327:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574218.646328:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574218.646330:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574218.646331:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574218.646333:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574218.646334:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574218.646336:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574218.646338:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574218.646339:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574218.646340:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574218.646342:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574218.646343:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574218.646346:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574218.646348:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574218.646350:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574218.646353:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393118080, portal 10
00000100:00000001:0.0:1788574218.646354:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574218.646355:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:0.0:1788574218.646360:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646364:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574218.646366:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574218.646367:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574218.646369:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393118080, offset 0
00000400:00000200:0.0:1788574218.646371:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574218.646374:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574218.646377:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574218.646379:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574218.646381:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574218.646382:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574218.646384:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574218.646387:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330178:330178:256:4294967295] 192.168.202.110@tcp LPNI seq info [330135:330135:8:4294967295]
00000400:00000200:0.0:1788574218.646390:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574218.646422:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574218.646426:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574218.646429:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574218.646432:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574218.646434:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1788574218.646437:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574218.646439:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:0.0:1788574218.646442:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574218.646443:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574218.646445:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574218.646448:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574218.646450:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574218.646451:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574218.646453:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574218.646455:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646461:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574218.646463:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646468:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574218.646470:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574218.646473:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574218.646477:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574218.646481:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574218.646483:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305822247
00000100:00000001:0.0:1788574218.646486:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305823161
00000100:00000001:0.0:1788574218.646488:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305822247
00000100:00000001:0.0:1788574218.646490:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305823161
00000100:00000001:0.0:1788574218.646492:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574218.646495:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574218.646497:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574218.646501:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574218/real 1788574218]  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574218.646511:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574218.646512:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574218.646514:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574218.646517:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574218.646519:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574218.646521:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574218.646525:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574218.646538:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646543:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574218.646544:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646545:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574218.646546:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574218.646549:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646553:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574218.646556:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646560:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574218.646562:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646563:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574218.646563:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646565:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574218.646566:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574218.646568:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574218.646570:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574218.646572:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574218.646574:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574218.646579:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574218.646582:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574218.646584:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646588:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574218.646589:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574218.646590:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574218.646591:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574218.646593:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574218.646595:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118208:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574218.646597:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574218.646599:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574218.646600:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574218.646601:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574218.646608:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574218.646610:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574218.646613:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574218.646615:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574218.646618:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574218.646620:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393118208, portal 4
00000100:00000001:0.0:1788574218.646621:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574218.646622:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:0.0:1788574218.646626:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646631:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574218.646633:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574218.646634:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574218.646636:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393118208, offset 0
00000400:00000200:0.0:1788574218.646638:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574218.646641:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574218.646644:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574218.646646:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574218.646648:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574218.646649:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574218.646651:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574218.646653:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330179:330179:256:4294967295] 192.168.202.110@tcp LPNI seq info [330136:330136:8:4294967295]
00000400:00000200:0.0:1788574218.646656:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574218.646660:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574218.646663:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574218.646665:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574218.646668:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574218.646670:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1788574218.646673:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574218.646675:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:0.0:1788574218.646677:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574218.646679:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574218.646681:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574218.646683:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574218.646685:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574218.646686:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574218.646688:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574218.646690:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646695:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574218.646697:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646702:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574218.646702:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574218.646705:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574218.646707:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574218.646709:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574218.646710:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305822393
00000100:00000001:0.0:1788574218.646712:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305823161
00000100:00000001:0.0:1788574218.646713:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305822393
00000100:00000001:0.0:1788574218.646715:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305823161
00000100:00000001:0.0:1788574218.646717:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574218.646718:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574218.646719:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574218.646721:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574218/real 1788574218]  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574218.646726:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574218.646727:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574218.646728:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574218.646729:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574218.646730:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574218.646731:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574218.646733:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574218.646738:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646742:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574218.646743:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646744:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574218.646745:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574218.646747:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646751:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574218.646754:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646759:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574218.646760:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646761:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574218.646762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646763:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574218.646764:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574218.646766:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574218.646768:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574218.646769:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574218.646771:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574218.646776:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118208:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574218.646778:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646779:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574218.646781:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393118080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646785:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574218.646786:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574218.646787:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574218.646789:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574218.646790:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574218.646791:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574218.646792:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574218.646793:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788574218.646795:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574218.646796:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574218.646797:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:0.0:1788574218.646799:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574218.646800:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646801:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574218.646803:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393118208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574218.646807:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574218.646808:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574218.646809:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574218.646810:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574218.646811:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574218.646812:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574218.646813:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574218.646814:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788574218.646815:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574218.646815:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574218.646816:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:0.0:1788574218.646818:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574218.646818:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646819:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574218.646822:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574218.646823:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574218.646824:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574218.646825:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574219.157482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574219.157490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574219.157492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574219.157498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574219.157502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574219.157506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574219.157509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574219.157512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574219.157516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574219.605454:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305823161, at 4305823168
00000100:00000040:0.0:1788574219.605465:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305823161, at 4305823168
00000100:00000010:1.0:1788574219.605467:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7240.
00000100:00000010:0.0:1788574219.605474:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788574220.053584:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574220.053588:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574220.053590:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574220.181492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574220.181499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574220.181501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574220.181504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574220.181507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574220.181510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574220.181512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574220.181514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574220.181516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574221.077577:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574221.077580:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574221.077583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574221.205460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574221.205467:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574221.205470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574221.205475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574221.205479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574221.205483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574221.205486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574221.205489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574221.205492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574222.229502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574222.229508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574222.229510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574222.229514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574222.229517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574222.229520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574222.229522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574222.229524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574222.229526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574223.125526:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574223.125530:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574223.125533:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574223.253455:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574223.253463:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574223.253466:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574223.253471:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574223.253475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574223.253479:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574223.253481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574223.253484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574223.253487:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574223.765731:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574223.765737:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574223.765742:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574223.765755:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.765775:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574223.765780:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574223.765787:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574223.765792:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574223.765798:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574223.765806:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393118336:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574223.765817:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574223.765823:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574223.765826:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574223.765828:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574223.765835:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574223.765839:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574223.765850:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574223.765855:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574223.765862:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574223.765870:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393118336, portal 25
00000100:00000001:0.0:1788574223.765874:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574223.765878:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:0.0:1788574223.765894:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.765906:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574223.765913:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574223.765917:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574223.765920:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393118336, offset 0
00000400:00000200:0.0:1788574223.765931:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574223.765942:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574223.765950:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574223.765957:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574223.765964:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574223.765970:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574223.765974:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574223.765981:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330180:330180:256:4294967295] 192.168.202.110@tcp LPNI seq info [330137:330137:8:4294967295]
00000400:00000200:0.0:1788574223.765987:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574223.765999:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574223.766006:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574223.766014:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574223.766023:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574223.766027:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574223.766036:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574223.766041:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574223.766050:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574223.766053:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574223.766059:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574223.766068:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574223.766076:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574223.766079:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574223.766084:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574223.766090:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766098:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574223.766101:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766106:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574223.766107:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574223.766111:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574223.766114:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574223.766116:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305828281
00000100:00000010:0.0:1788574223.766128:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574223.766136:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574223.766141:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305828281
00000100:00000010:0.0:1788574223.766143:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7180.
00000100:00000001:0.0:1788574223.766145:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574223.766147:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574223.766149:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574223.766151:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574223.766155:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574223/real 1788574223]  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574223.766166:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574223.766168:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574223.766169:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574223.766171:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574223.766172:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574223.766174:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574223.766176:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574223.766182:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766186:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574223.766187:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766188:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574223.766189:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574223.766192:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766197:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574223.766200:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766205:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574223.766207:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766208:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574223.766209:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766210:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574223.766212:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574223.766213:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574223.766216:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574223.766217:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574223.766220:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574223.766227:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393118336:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574223.766231:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766232:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574223.766234:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393118336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766238:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574223.766240:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574223.766241:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574223.766244:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574223.766245:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574223.766246:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574223.766247:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574223.766249:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:0.0:1788574223.766252:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574223.766253:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574223.766255:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:0.0:1788574223.766257:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574223.766258:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766260:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766265:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574223.766267:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574223.766268:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574223.766269:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574223.766270:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574223.766272:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766277:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574223.766278:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574223.766280:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574223.766281:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574223.766282:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574223.766284:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118464:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574223.766287:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574223.766288:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574223.766289:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574223.766290:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574223.766292:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574223.766294:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574223.766296:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574223.766298:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574223.766300:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574223.766302:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393118464, portal 10
00000100:00000001:0.0:1788574223.766303:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574223.766304:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:0.0:1788574223.766309:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766313:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574223.766315:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574223.766316:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574223.766318:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393118464, offset 0
00000400:00000200:0.0:1788574223.766321:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574223.766323:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574223.766325:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574223.766327:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574223.766329:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574223.766330:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574223.766332:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574223.766334:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330181:330181:256:4294967295] 192.168.202.110@tcp LPNI seq info [330138:330138:8:4294967295]
00000400:00000200:0.0:1788574223.766337:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574223.766341:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574223.766344:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574223.766346:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574223.766349:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574223.766351:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574223.766354:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574223.766356:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574223.766358:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574223.766359:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574223.766361:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574223.766364:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574223.766366:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574223.766368:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574223.766370:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574223.766374:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766382:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574223.766385:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766417:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574223.766419:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574223.766424:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574223.766427:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574223.766430:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574223.766432:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305827367
00000100:00000001:0.0:1788574223.766434:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305828281
00000100:00000001:0.0:1788574223.766436:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305827367
00000100:00000001:0.0:1788574223.766437:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305828281
00000100:00000001:0.0:1788574223.766439:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574223.766440:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574223.766441:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574223.766444:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574223/real 1788574223]  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574223.766450:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574223.766451:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574223.766452:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574223.766454:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574223.766455:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574223.766457:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574223.766459:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574223.766464:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766469:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574223.766470:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766471:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574223.766472:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574223.766474:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766478:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574223.766480:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766485:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574223.766486:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766487:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574223.766488:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766490:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574223.766491:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574223.766493:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574223.766494:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574223.766496:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574223.766498:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574223.766503:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118464:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574223.766506:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574223.766508:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766512:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574223.766513:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574223.766514:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574223.766516:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574223.766517:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574223.766519:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118592:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574223.766520:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574223.766521:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574223.766523:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574223.766524:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574223.766526:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574223.766528:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574223.766530:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574223.766532:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574223.766534:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574223.766536:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393118592, portal 4
00000100:00000001:0.0:1788574223.766537:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574223.766538:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:0.0:1788574223.766542:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766546:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574223.766547:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574223.766549:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574223.766550:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393118592, offset 0
00000400:00000200:0.0:1788574223.766553:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574223.766556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574223.766558:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574223.766560:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574223.766562:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574223.766564:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574223.766565:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574223.766567:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330182:330182:256:4294967295] 192.168.202.110@tcp LPNI seq info [330139:330139:8:4294967295]
00000400:00000200:0.0:1788574223.766570:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574223.766574:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574223.766577:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574223.766579:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574223.766582:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574223.766583:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574223.766586:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574223.766588:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574223.766607:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574223.766608:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574223.766610:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574223.766613:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574223.766615:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574223.766616:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574223.766618:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574223.766621:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766626:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574223.766628:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766632:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574223.766633:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574223.766636:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574223.766638:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574223.766640:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574223.766641:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305827513
00000100:00000001:0.0:1788574223.766643:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305828281
00000100:00000001:0.0:1788574223.766645:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305827513
00000100:00000001:0.0:1788574223.766646:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305828281
00000100:00000001:0.0:1788574223.766648:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574223.766649:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574223.766651:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574223.766653:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574223/real 1788574223]  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574223.766658:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574223.766658:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574223.766659:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574223.766661:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574223.766662:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574223.766663:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574223.766665:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574223.766670:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766674:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574223.766675:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766676:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574223.766677:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574223.766679:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766683:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574223.766685:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766689:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574223.766691:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766692:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574223.766692:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766694:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574223.766695:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574223.766697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574223.766699:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574223.766701:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574223.766703:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574223.766708:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118592:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574223.766710:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766711:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574223.766713:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393118464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766717:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574223.766718:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574223.766719:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574223.766721:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574223.766722:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574223.766723:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574223.766724:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574223.766726:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:0.0:1788574223.766727:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574223.766728:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574223.766729:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:0.0:1788574223.766731:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574223.766732:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766733:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574223.766735:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393118592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574223.766739:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574223.766740:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574223.766741:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574223.766742:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574223.766743:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574223.766744:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574223.766745:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574223.766746:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788574223.766748:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574223.766748:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574223.766749:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:0.0:1788574223.766751:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574223.766752:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766753:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574223.766755:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574223.766756:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574223.766757:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574223.766758:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574224.149592:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574224.149596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574224.149599:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574224.277500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574224.277508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574224.277511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574224.277517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574224.277520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574224.277524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574224.277526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574224.277527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574224.277530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574224.725503:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305828281, at 4305828288
00000100:00000040:1.0:1788574224.725512:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305828281, at 4305828288
00000100:00000010:0.0:1788574224.725516:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574224.725520:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7180.
00000100:00000001:1.0:1788574225.301490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574225.301498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574225.301515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574225.301521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574225.301524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574225.301528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574225.301531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574225.301535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574225.301538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574226.197573:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574226.197577:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574226.197580:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574226.325516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574226.325524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574226.325527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574226.325533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574226.325537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574226.325541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574226.325544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574226.325547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574226.325550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574227.221582:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574227.221585:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574227.221588:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574227.349495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574227.349500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574227.349502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574227.349506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574227.349509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574227.349512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574227.349514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574227.349517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574227.349520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574228.373477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574228.373483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574228.373485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574228.373489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574228.373492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574228.373495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574228.373497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574228.373499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574228.373501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574228.885735:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574228.885740:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574228.885744:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574228.885753:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.885770:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574228.885773:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574228.885778:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574228.885781:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574228.885785:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574228.885793:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393118720:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574228.885801:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574228.885807:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574228.885810:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574228.885814:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574228.885822:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574228.885825:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574228.885834:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574228.885839:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574228.885848:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574228.885856:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393118720, portal 25
00000100:00000001:0.0:1788574228.885860:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574228.885863:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:0.0:1788574228.885872:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.885886:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574228.885893:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574228.885897:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574228.885904:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393118720, offset 0
00000400:00000200:0.0:1788574228.885918:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574228.885931:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574228.885944:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574228.885954:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574228.885960:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574228.885965:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574228.885969:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574228.885975:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330183:330183:256:4294967295] 192.168.202.110@tcp LPNI seq info [330140:330140:8:4294967295]
00000400:00000200:0.0:1788574228.885985:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574228.885997:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574228.886005:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574228.886014:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574228.886024:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574228.886030:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788574228.886038:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574228.886045:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788574228.886057:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574228.886063:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574228.886072:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574228.886084:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574228.886093:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574228.886096:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574228.886101:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574228.886105:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886114:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574228.886116:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886122:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574228.886124:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574228.886128:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574228.886132:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574228.886134:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305833401
00000100:00000010:0.0:1788574228.886138:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574228.886143:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574228.886148:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305833401
00000100:00000010:0.0:1788574228.886150:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7d20.
00000100:00000001:0.0:1788574228.886153:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574228.886155:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574228.886156:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574228.886159:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574228.886161:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574228/real 1788574228]  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574228.886169:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574228.886170:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574228.886172:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574228.886174:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574228.886176:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574228.886177:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574228.886179:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574228.886190:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886194:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574228.886195:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886197:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574228.886198:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574228.886201:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886206:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574228.886209:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886214:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574228.886216:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886217:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574228.886217:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886219:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574228.886221:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574228.886223:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574228.886225:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574228.886227:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574228.886230:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574228.886236:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393118720:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574228.886239:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886240:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574228.886242:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393118720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886250:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574228.886252:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574228.886255:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574228.886257:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574228.886258:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574228.886259:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574228.886261:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574228.886263:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:0.0:1788574228.886265:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574228.886267:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574228.886268:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:0.0:1788574228.886270:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574228.886271:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886273:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886278:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574228.886279:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574228.886281:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574228.886282:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574228.886283:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574228.886286:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886290:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574228.886291:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574228.886321:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574228.886323:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574228.886324:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574228.886327:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574228.886330:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574228.886332:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574228.886333:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574228.886334:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574228.886337:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574228.886338:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574228.886341:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574228.886343:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574228.886346:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574228.886348:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393118848, portal 10
00000100:00000001:0.0:1788574228.886350:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574228.886351:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:0.0:1788574228.886355:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886360:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574228.886362:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574228.886363:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574228.886365:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393118848, offset 0
00000400:00000200:0.0:1788574228.886367:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574228.886370:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574228.886373:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574228.886375:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574228.886377:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574228.886379:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574228.886380:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574228.886383:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330184:330184:256:4294967295] 192.168.202.110@tcp LPNI seq info [330141:330141:8:4294967295]
00000400:00000200:0.0:1788574228.886385:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574228.886390:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574228.886415:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574228.886418:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574228.886421:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574228.886423:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788574228.886427:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574228.886429:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788574228.886432:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574228.886433:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574228.886436:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574228.886438:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574228.886440:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574228.886442:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574228.886444:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574228.886447:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886452:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574228.886454:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886459:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574228.886460:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574228.886462:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574228.886465:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574228.886467:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574228.886469:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305832487
00000100:00000001:0.0:1788574228.886471:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305833401
00000100:00000001:0.0:1788574228.886472:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305832487
00000100:00000001:0.0:1788574228.886474:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305833401
00000100:00000001:0.0:1788574228.886475:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574228.886477:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574228.886478:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574228.886480:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574228/real 1788574228]  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574228.886485:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574228.886486:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574228.886487:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574228.886489:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574228.886490:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574228.886492:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574228.886494:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574228.886499:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886508:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574228.886509:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886510:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574228.886511:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574228.886513:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886517:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574228.886519:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886524:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574228.886525:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886526:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574228.886527:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886528:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574228.886529:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574228.886531:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574228.886533:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574228.886534:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574228.886537:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574228.886542:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574228.886545:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574228.886547:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886551:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574228.886552:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574228.886554:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574228.886555:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574228.886556:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574228.886558:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574228.886560:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574228.886562:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574228.886563:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574228.886564:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574228.886566:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574228.886567:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574228.886569:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574228.886571:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574228.886573:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574228.886575:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393118976, portal 4
00000100:00000001:0.0:1788574228.886576:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574228.886577:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:0.0:1788574228.886581:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886585:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574228.886587:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574228.886588:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574228.886590:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393118976, offset 0
00000400:00000200:0.0:1788574228.886592:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574228.886595:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574228.886598:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574228.886601:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574228.886603:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574228.886606:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574228.886608:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574228.886611:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330185:330185:256:4294967295] 192.168.202.110@tcp LPNI seq info [330142:330142:8:4294967295]
00000400:00000200:0.0:1788574228.886616:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574228.886623:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574228.886628:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574228.886632:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574228.886636:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574228.886638:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788574228.886642:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574228.886645:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788574228.886649:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574228.886651:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788574228.886654:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574228.886657:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574228.886660:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4908
00000400:00000010:0.0:1788574228.886662:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4908.
00000100:00000001:0.0:1788574228.886665:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574228.886669:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886678:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574228.886681:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886686:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574228.886687:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574228.886690:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574228.886692:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574228.886695:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574228.886697:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305832633
00000100:00000001:0.0:1788574228.886700:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305833401
00000100:00000001:0.0:1788574228.886702:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305832633
00000100:00000001:0.0:1788574228.886703:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305833401
00000100:00000001:0.0:1788574228.886705:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574228.886706:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574228.886708:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574228.886710:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574228/real 1788574228]  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574228.886716:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574228.886717:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574228.886718:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574228.886720:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574228.886721:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574228.886722:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574228.886724:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574228.886730:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886734:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574228.886735:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886736:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574228.886737:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574228.886739:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886743:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574228.886745:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886749:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574228.886751:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886752:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574228.886753:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886754:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574228.886756:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574228.886757:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574228.886759:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574228.886761:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574228.886764:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574228.886769:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393118976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574228.886771:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886773:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574228.886775:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393118848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886778:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574228.886780:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574228.886781:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574228.886782:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574228.886783:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574228.886784:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574228.886786:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574228.886787:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788574228.886789:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574228.886790:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574228.886791:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:0.0:1788574228.886793:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574228.886794:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886795:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574228.886797:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393118976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574228.886801:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574228.886802:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574228.886803:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574228.886804:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574228.886804:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574228.886805:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574228.886806:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574228.886807:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:0.0:1788574228.886808:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574228.886810:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574228.886811:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:0.0:1788574228.886812:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574228.886813:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886814:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574228.886817:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574228.886818:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574228.886819:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574228.886820:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574229.269612:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574229.269615:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574229.269618:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574229.397470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574229.397476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574229.397477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574229.397481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574229.397483:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574229.397486:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574229.397488:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574229.397490:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574229.397492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574229.845453:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305833401, at 4305833408
00000100:00000040:0.0:1788574229.845461:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305833401, at 4305833408
00000100:00000010:1.0:1788574229.845468:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d20.
00000100:00000010:0.0:1788574229.845471:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788574230.293582:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574230.293585:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574230.293589:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574230.421475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574230.421481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574230.421483:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574230.421486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574230.421489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574230.421493:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574230.421495:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574230.421497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574230.421499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574231.445494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574231.445501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574231.445503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574231.445509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574231.445513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574231.445516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574231.445519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574231.445522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574231.445525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574232.341539:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574232.341544:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574232.341547:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574233.365574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574233.365579:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574233.365583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574234.005488:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574234.005497:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574234.005499:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574234.005503:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574234.005507:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574234.005512:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11165
00000400:00000200:0.0:1788574234.005520:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574234.005525:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574234.005527:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574234.005530:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005532:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574234.005534:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005536:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574234.005539:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574234.005541:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574234.005543:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574234.005545:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574234.005547:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574234.005549:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574234.005551:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005552:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574234.005553:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005554:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574234.005556:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574234.005559:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005560:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574234.005563:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574234.005565:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574234.005567:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574234.005569:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574234.005570:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574234.005577:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788574234.005581:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574234.005584:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574234.005586:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574234.005589:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574234.005592:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574234.005596:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574234.005604:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788574234.005610:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574234.005620:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393119104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574234.005638:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574234.005641:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574234.005645:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393119104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574234.005663:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574234.005666:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574234.005672:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574234.005673:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574234.005674:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574234.005676:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574234.005678:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11165
00000400:00000200:0.0:1788574234.005682:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574234.005685:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574234.005687:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574234.005689:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005691:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574234.005693:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005695:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574234.005697:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574234.005698:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574234.005703:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574234.005705:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574234.005710:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574234.005711:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574234.005716:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005718:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574234.005722:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005723:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574234.005725:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574234.005730:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005734:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574234.005739:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574234.005746:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574234.005750:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574234.005754:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574234.005757:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574234.005762:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574234.005765:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574234.005770:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788574234.005774:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574234.005778:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574234.005781:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574234.005783:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574234.005787:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574234.005791:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574234.005796:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788574234.005803:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574234.005816:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393119232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574234.005832:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574234.005836:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574234.005847:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393119232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574234.005873:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574234.005879:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574234.005884:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574234.005887:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574234.005890:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574234.005894:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574234.005900:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11165
00000400:00000200:0.0:1788574234.005906:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574234.005914:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574234.005916:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574234.005918:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005923:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574234.005928:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005932:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574234.005936:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574234.005938:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574234.005939:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574234.005941:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574234.005942:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574234.005943:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574234.005946:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005948:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574234.005951:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005954:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574234.005957:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574234.005965:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574234.005970:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574234.005974:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574234.005977:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574234.005981:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574234.005984:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574234.005986:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574234.005988:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574234.005991:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574234.005997:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574234.006000:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574234.006003:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574234.006006:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574234.006008:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574234.006011:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574234.006014:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574234.006020:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788574234.006024:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574234.006031:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393119360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1788574234.006054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574234.006060:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1788574234.006065:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574234.006069:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574234.006077:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393119360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574234.006084:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574234.006087:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574234.006090:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574234.006097:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11175)
00000800:00000200:2.1:1788574234.006370:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574234.006372:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574234.006639:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574234.006640:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1788574234.965494:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305838521, at 4305838528
00000100:00000040:1.0:1788574234.965494:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305838521, at 4305838528
00000100:00000010:1.0:1788574234.965509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000100:00000010:0.0:1788574234.965515:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000020:00000001:0.0:1788574235.413602:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574235.413606:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574235.413609:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574236.437643:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574236.437646:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574236.437649:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574238.485593:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574238.485596:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574238.485599:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574239.125451:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574239.125460:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574239.125462:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574239.125465:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574239.125469:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574239.125475:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11170
00000400:00000200:0.0:1788574239.125484:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574239.125490:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574239.125492:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574239.125494:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125497:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574239.125499:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125502:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574239.125505:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574239.125506:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574239.125508:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574239.125510:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574239.125512:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574239.125514:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574239.125515:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125517:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574239.125518:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125519:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574239.125521:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574239.125523:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125525:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574239.125528:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574239.125530:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574239.125533:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574239.125534:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574239.125536:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574239.125544:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574239.125546:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574239.125549:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574239.125551:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574239.125554:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574239.125556:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574239.125559:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574239.125566:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574239.125572:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574239.125582:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393119488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574239.125594:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574239.125598:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574239.125604:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393119488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574239.125628:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574239.125632:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574239.125636:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574239.125637:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574239.125638:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574239.125641:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574239.125643:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11170
00000400:00000200:0.0:1788574239.125646:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574239.125649:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574239.125651:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574239.125653:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125655:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574239.125658:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125660:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574239.125662:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574239.125663:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574239.125664:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574239.125665:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574239.125667:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574239.125669:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574239.125672:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125675:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574239.125677:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125682:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574239.125684:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574239.125688:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125690:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574239.125692:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574239.125697:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574239.125702:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574239.125706:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574239.125711:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574239.125714:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574239.125718:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574239.125722:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788574239.125725:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574239.125730:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574239.125733:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574239.125738:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574239.125741:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574239.125744:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574239.125749:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788574239.125755:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574239.125763:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393119616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574239.125774:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574239.125777:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574239.125780:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393119616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574239.125794:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574239.125796:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574239.125801:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574239.125805:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574239.125807:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574239.125809:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574239.125811:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11170
00000400:00000200:0.0:1788574239.125814:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574239.125815:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574239.125817:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574239.125819:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125825:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574239.125827:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125828:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574239.125830:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574239.125832:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574239.125835:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574239.125838:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574239.125839:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574239.125841:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574239.125843:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125849:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574239.125853:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125856:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574239.125860:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574239.125863:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574239.125867:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574239.125868:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574239.125872:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574239.125875:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574239.125879:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574239.125882:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574239.125887:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574239.125890:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574239.125893:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788574239.125897:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574239.125901:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574239.125903:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574239.125905:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574239.125907:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574239.125910:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574239.125914:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574239.125919:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574239.125924:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393119744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574239.125931:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574239.125932:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574239.125939:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393119744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574239.125946:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574239.125949:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574239.125956:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574239.125960:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11180)
00000800:00000200:2.1:1788574239.126066:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574239.126070:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574239.126590:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574239.126593:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574239.126929:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574239.126932:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:1.0:1788574239.509646:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574239.509650:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574239.509652:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574240.085508:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305843641, at 4305843648
00000100:00000040:1.0:1788574240.085509:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305843641, at 4305843648
00000100:00000010:0.0:1788574240.085527:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000100:00000010:1.0:1788574240.085537:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000020:00000001:0.0:1788574241.557609:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574241.557616:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574241.557620:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574242.581596:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574242.581600:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574242.581603:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788574244.245706:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574244.245712:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574244.245716:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574244.245726:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393119872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574244.245746:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574244.245750:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574244.245757:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574244.245759:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574244.245762:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574244.245770:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393119872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574244.245776:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574244.245781:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574244.245784:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574244.245788:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574244.245794:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574244.245798:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574244.245805:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574244.245810:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574244.245817:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574244.245825:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393119872, portal 25
00000100:00000001:0.0:1788574244.245830:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574244.245834:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:0.0:1788574244.245844:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393119872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574244.245876:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574244.245906:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574244.245911:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4908.
00000100:00000200:0.0:1788574244.245916:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393119872, offset 0
00000400:00000200:0.0:1788574244.245929:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574244.245942:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574244.245954:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574244.245964:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574244.245973:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574244.245979:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574244.245987:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574244.245992:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330192:330192:256:4294967295] 192.168.202.110@tcp LPNI seq info [330149:330149:8:4294967295]
00000400:00000200:0.0:1788574244.246002:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574244.246017:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574244.246025:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574244.246032:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788574244.246043:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574244.246052:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574244.246067:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574244.246072:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574244.246075:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305848761
00000100:00000010:0.0:1788574244.246081:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574244.246092:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574244.246102:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305848761
00000100:00000010:0.0:1788574244.246110:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7b40.
00000100:00000001:0.0:1788574244.246116:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574244.246124:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574244.246131:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574244.246136:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574244.246140:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574244.246149:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574244.246152:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788574244.246155:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574244.246158:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574244.246161:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574244.246167:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393120000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574244.246184:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574244.246186:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574244.246190:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574244.246193:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574244.246199:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574244.246205:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393120000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574244.246211:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574244.246214:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574244.246216:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574244.246217:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574244.246222:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574244.246225:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574244.246230:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574244.246234:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dac60.
00000400:00000010:0.0:1788574244.246239:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574244.246242:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393120000, portal 10
00000100:00000001:0.0:1788574244.246244:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574244.246245:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:0.0:1788574244.246252:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393120000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574244.246257:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574244.246258:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574244.246261:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4f68.
00000100:00000200:0.0:1788574244.246262:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393120000, offset 0
00000400:00000200:0.0:1788574244.246265:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574244.246268:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574244.246270:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574244.246272:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574244.246274:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574244.246276:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574244.246278:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574244.246280:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330193:330193:255:4294967295] 192.168.202.110@tcp LPNI seq info [330150:330150:7:4294967295]
00000400:00000200:0.0:1788574244.246283:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574244.246287:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574244.246290:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574244.246293:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9900.
00000800:00000200:0.0:1788574244.246296:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574244.246298:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574244.246300:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574244.246302:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574244.246305:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574244.246306:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305847846
00000100:00000001:0.0:1788574244.246308:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305848761
00000100:00000001:0.0:1788574244.246309:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305847846
00000100:00000001:0.0:1788574244.246311:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305848761
00000100:00000001:0.0:1788574244.246313:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574244.246314:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574244.246316:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574244.246318:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574244.246320:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574244.246321:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788574244.246323:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574244.246325:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574244.246326:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574244.246330:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393120128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574244.246338:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574244.246339:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574244.246341:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574244.246343:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574244.246345:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574244.246348:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393120128:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574244.246351:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574244.246353:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574244.246355:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574244.246356:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574244.246360:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574244.246361:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574244.246365:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574244.246368:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574244.246371:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4220.
00000100:00000200:0.0:1788574244.246373:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393120128, portal 4
00000100:00000001:0.0:1788574244.246375:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574244.246376:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:0.0:1788574244.246381:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393120128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574244.246388:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574244.246391:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574244.246415:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4000.
00000100:00000200:0.0:1788574244.246419:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393120128, offset 0
00000400:00000200:0.0:1788574244.246423:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574244.246428:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574244.246431:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574244.246434:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574244.246435:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574244.246437:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574244.246439:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574244.246441:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330194:330194:254:4294967295] 192.168.202.110@tcp LPNI seq info [330151:330151:6:4294967295]
00000400:00000200:0.0:1788574244.246444:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574244.246449:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574244.246454:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574244.246457:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9400.
00000800:00000200:0.0:1788574244.246462:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574244.246466:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574244.246469:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574244.246472:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574244.246476:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574244.246478:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305847993
00000100:00000001:0.0:1788574244.246481:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305848761
00000100:00000001:0.0:1788574244.246483:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305847993
00000100:00000001:0.0:1788574244.246485:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305848761
00000100:00000001:0.0:1788574244.246487:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574244.246489:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574244.246492:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574244.246494:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574244.246497:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574244.246499:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788574244.246500:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574244.246502:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574244.246503:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574244.246505:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574244.629643:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574244.629649:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574244.629653:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574245.205468:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305848761, at 4305848768
00000100:00000040:1.0:1788574245.205468:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305848761, at 4305848768
00000100:00000010:1.0:1788574245.205484:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7b40.
00000100:00000010:0.0:1788574245.205486:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788574245.653564:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574245.653568:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574245.653572:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788574247.318111:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574247.318118:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574247.318121:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574247.318131:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574244/real 1788574247]  req@ffff895348e02680 x1875444393120128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574247.318146:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574247.318148:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574247.318152:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da9f8.
00000400:00000200:0.0:1788574247.318157:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4220
00000400:00000010:0.0:1788574247.318159:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4220.
00000100:00000001:0.0:1788574247.318163:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574247.318167:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393120128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574247.318179:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393120128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318187:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574247.318190:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318193:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574247.318196:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574247.318201:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393120128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318209:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574247.318216:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393120128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318226:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574247.318230:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318232:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574247.318233:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318238:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574247.318242:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574247.318247:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574247.318254:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574247.318257:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574247.318263:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393120128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574247.318273:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393120128:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574247.318279:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574247.318284:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574244/real 1788574247]  req@ffff895348e00700 x1875444393120000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574247.318292:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574247.318294:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574247.318296:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dac60.
00000400:00000200:0.0:1788574247.318298:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574247.318300:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574247.318302:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574247.318304:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393120000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574247.318312:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393120000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318319:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574247.318321:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318323:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574247.318324:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574247.318329:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393120000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318335:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574247.318340:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393120000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318348:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574247.318351:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318353:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574247.318354:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318356:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574247.318358:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574247.318361:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574247.318365:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574247.318367:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574247.318372:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393120000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574247.318381:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393120000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574247.318385:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574247.318390:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574244/real 1788574247]  req@ffff895348e03480 x1875444393119872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574247.318422:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574247.318424:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574247.318426:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574247.318429:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574247.318431:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574247.318435:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574247.318440:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393119872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574247.318449:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393119872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318456:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574247.318457:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318460:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574247.318461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574247.318465:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393119872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318471:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574247.318474:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393119872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318479:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574247.318481:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318482:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574247.318483:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318484:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574247.318493:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574247.318496:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574247.318500:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574247.318502:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574247.318506:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393119872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574247.318514:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393119872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574247.318520:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318522:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574247.318526:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393120128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318534:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574247.318538:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574247.318543:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574247.318548:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574247.318552:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574247.318554:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574247.318557:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574247.318559:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788574247.318562:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574247.318564:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574247.318567:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:0.0:1788574247.318571:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574247.318572:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318575:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574247.318579:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393120000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318586:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574247.318587:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574247.318589:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574247.318592:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574247.318593:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574247.318595:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574247.318597:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574247.318598:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788574247.318601:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574247.318602:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574247.318604:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:0.0:1788574247.318607:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574247.318608:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318610:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574247.318614:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393119872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574269 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574247.318621:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574247.318623:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574247.318625:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574247.318627:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574247.318628:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574247.318630:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574247.318631:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574247.318633:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788574247.318636:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574247.318637:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574247.318639:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:0.0:1788574247.318642:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574247.318643:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318646:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574247.318654:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574247.318657:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574247.318659:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574247.318661:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574247.701538:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574247.701541:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574247.701545:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574248.725567:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574248.725570:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574248.725573:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574249.365484:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574249.365493:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574249.365495:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574249.365498:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574249.365503:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574249.365508:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11180
00000400:00000200:0.0:1788574249.365516:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574249.365529:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574249.365532:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574249.365535:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365540:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574249.365543:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365545:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574249.365548:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574249.365549:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574249.365551:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574249.365554:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574249.365555:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574249.365557:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574249.365559:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365561:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574249.365562:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365564:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574249.365566:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574249.365569:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365571:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574249.365573:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574249.365575:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574249.365578:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574249.365579:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574249.365581:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574249.365587:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788574249.365589:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574249.365592:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574249.365594:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574249.365596:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574249.365598:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574249.365602:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574249.365607:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574249.365613:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574249.365620:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393120256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574249.365629:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574249.365634:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574249.365639:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393120256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574249.365665:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574249.365668:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574249.365672:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574249.365674:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574249.365675:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574249.365678:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574249.365681:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11180
00000400:00000200:0.0:1788574249.365686:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574249.365701:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574249.365705:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574249.365712:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365718:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574249.365725:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365728:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574249.365733:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574249.365735:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574249.365739:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574249.365743:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574249.365748:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574249.365754:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574249.365758:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365760:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574249.365762:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365764:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574249.365766:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574249.365769:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365772:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574249.365776:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574249.365783:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574249.365788:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574249.365793:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574249.365799:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574249.365824:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574249.365825:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574249.365829:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574249.365832:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574249.365834:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574249.365836:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574249.365838:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574249.365841:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574249.365844:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574249.365851:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574249.365857:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574249.365867:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393120384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574249.365891:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574249.365896:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574249.365905:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393120384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574249.365921:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574249.365926:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574249.365932:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574249.365935:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574249.365938:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574249.365945:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574249.365952:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11180
00000400:00000200:0.0:1788574249.365963:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574249.365970:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574249.365975:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574249.365981:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.365988:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574249.366008:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.366012:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574249.366017:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574249.366020:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574249.366024:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574249.366028:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574249.366032:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574249.366038:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574249.366047:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.366052:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574249.366058:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574249.366065:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574249.366072:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574249.366078:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574249.366082:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574249.366087:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574249.366093:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574249.366099:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574249.366102:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574249.366106:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574249.366111:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574249.366113:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574249.366119:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788574249.366125:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574249.366130:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574249.366133:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574249.366138:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574249.366141:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574249.366145:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574249.366151:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:0.0:1788574249.366154:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000800:00000200:2.1:1788574249.366159:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1788574249.366161:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574249.366172:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393120512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574249.366188:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574249.366193:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574249.366201:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393120512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574249.366218:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574249.366225:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574249.366233:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574249.366242:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11190)
00000800:00000200:2.1:1788574249.366667:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574249.366670:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788574249.367002:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574249.367005:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:0.0:1788574250.325481:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305853881, at 4305853888
00000100:00000040:1.0:1788574250.325481:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305853881, at 4305853888
00000100:00000010:0.0:1788574250.325496:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000100:00000010:1.0:1788574250.325499:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00000020:00000001:0.0:1788574250.773543:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574250.773547:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574250.773550:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574251.797558:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574251.797563:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574251.797566:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788574252.437524:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574252.437531:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574252.437535:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574252.437725:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574252.437744:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574252.437754:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574252.437789:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574252.437799:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1788574253.845538:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574253.845541:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574253.845545:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788574254.485522:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574254.485534:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574254.485537:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574254.485542:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574254.485547:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574254.485555:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11185
00000400:00000200:1.0:1788574254.485565:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574254.485573:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574254.485577:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574254.485581:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485586:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574254.485589:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485592:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574254.485597:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574254.485599:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574254.485602:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574254.485605:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574254.485608:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574254.485611:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574254.485615:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485617:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574254.485625:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485627:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574254.485631:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574254.485636:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485639:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574254.485642:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574254.485646:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574254.485649:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574254.485651:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574254.485655:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574254.485665:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788574254.485668:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574254.485672:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574254.485674:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574254.485678:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574254.485681:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574254.485685:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574254.485694:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788574254.485701:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574254.485710:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393120640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574254.485761:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574254.485766:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574254.485772:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393120640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574254.485796:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574254.485799:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574254.485804:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574254.485805:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574254.485807:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574254.485811:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574254.485814:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11185
00000400:00000200:1.0:1788574254.485819:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574254.485824:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574254.485827:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574254.485831:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485835:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574254.485841:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485846:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574254.485851:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574254.485854:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574254.485858:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574254.485862:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574254.485867:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574254.485873:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574254.485882:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485887:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574254.485893:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485898:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574254.485904:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574254.485911:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574254.485917:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574254.485921:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574254.485928:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574254.485936:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574254.485939:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574254.485944:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574254.485947:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574254.485951:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574254.485957:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788574254.485962:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574254.485968:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574254.485973:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574254.485979:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574254.485983:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574254.485986:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574254.485994:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788574254.486001:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574254.486013:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393120768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574254.486030:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574254.486035:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574254.486044:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393120768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574254.486067:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574254.486074:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574254.486081:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574254.486084:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574254.486088:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574254.486094:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574254.486102:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11185
00000400:00000200:1.0:1788574254.486110:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574254.486118:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574254.486122:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574254.486130:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.1:1788574254.486131:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574254.486134:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788574254.486136:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:1.0:1788574254.486141:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.486147:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574254.486152:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574254.486156:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574254.486160:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574254.486166:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574254.486170:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574254.486174:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574254.486183:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.486188:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574254.486194:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574254.486200:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574254.486207:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574254.486213:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574254.486218:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574254.486223:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574254.486227:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574254.486232:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574254.486235:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574254.486241:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574254.486246:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574254.486248:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574254.486253:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788574254.486260:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574254.486267:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574254.486270:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574254.486274:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574254.486277:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574254.486280:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574254.486288:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788574254.486294:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574254.486301:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393120896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574254.486318:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574254.486324:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574254.486331:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393120896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574254.486344:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574254.486350:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574254.486358:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574254.486367:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11195)
00000800:00000200:2.1:1788574254.486631:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574254.486632:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788574254.486882:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574254.486884:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:1.0:1788574254.869592:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574254.869596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574254.869600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574255.445439:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305859001, at 4305859008
00000100:00000010:1.0:1788574255.445452:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00000100:00000040:0.0:1788574255.445471:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305859001, at 4305859008
00000100:00000010:0.0:1788574255.445480:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000020:00000001:0.0:1788574256.917578:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574256.917582:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574256.917586:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574257.941607:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574257.941611:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574257.941614:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788574259.605495:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574259.605505:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574259.605508:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574259.605512:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574259.605518:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574259.605526:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11190
00000400:00000200:1.0:1788574259.605536:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574259.605543:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574259.605547:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574259.605551:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605555:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574259.605558:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605561:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574259.605566:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574259.605568:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574259.605571:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574259.605574:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574259.605577:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574259.605580:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574259.605583:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605586:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574259.605588:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605591:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574259.605594:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574259.605597:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605600:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574259.605603:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574259.605606:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574259.605609:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574259.605612:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574259.605614:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574259.605623:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788574259.605627:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574259.605631:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574259.605633:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574259.605635:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574259.605638:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574259.605642:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574259.605679:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574259.605687:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574259.605696:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393121024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574259.605706:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574259.605711:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574259.605715:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393121024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574259.605734:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574259.605737:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574259.605741:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574259.605743:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574259.605744:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574259.605748:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574259.605750:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11190
00000400:00000200:1.0:1788574259.605755:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574259.605758:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574259.605761:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574259.605765:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605768:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574259.605771:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605773:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574259.605776:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574259.605781:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574259.605784:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574259.605790:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574259.605795:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574259.605802:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574259.605809:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605816:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574259.605820:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605826:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574259.605832:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574259.605840:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574259.605844:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574259.605849:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574259.605857:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574259.605864:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574259.605869:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574259.605876:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574259.605881:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574259.605885:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574259.605891:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788574259.605897:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574259.605902:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574259.605906:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574259.605911:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574259.605915:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574259.605919:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574259.605927:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574259.605932:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574259.605942:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393121152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574259.605956:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574259.605961:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574259.605970:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393121152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574259.605986:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574259.605992:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574259.606002:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574259.606005:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574259.606009:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574259.606013:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574259.606018:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11190
00000400:00000200:1.0:1788574259.606033:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574259.606039:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574259.606045:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574259.606053:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.606057:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788574259.606058:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574259.606062:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:1.0:1788574259.606063:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.606069:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574259.606074:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574259.606076:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574259.606079:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574259.606083:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574259.606088:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574259.606092:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574259.606100:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.606105:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574259.606111:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574259.606117:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574259.606122:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574259.606130:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574259.606138:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574259.606143:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574259.606148:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574259.606153:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574259.606159:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574259.606163:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574259.606167:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574259.606170:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574259.606177:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788574259.606180:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574259.606182:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574259.606183:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574259.606186:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574259.606190:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574259.606194:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574259.606201:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574259.606207:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574259.606217:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393121280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574259.606233:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574259.606238:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574259.606247:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393121280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574259.606274:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574259.606280:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574259.606288:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574259.606297:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11200)
00000800:00000200:2.1:1788574259.606514:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574259.606515:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788574259.606743:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574259.606745:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:0.0:1788574259.989558:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574259.989563:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574259.989566:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574260.565516:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305864121, at 4305864128
00000100:00000040:0.0:1788574260.565523:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305864121, at 4305864128
00000100:00000010:1.0:1788574260.565533:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00000100:00000010:0.0:1788574260.565545:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000020:00000001:1.0:1788574261.013534:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574261.013537:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574261.013540:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574263.061560:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574263.061564:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574263.061567:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574264.085739:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574264.085744:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574264.085747:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788574264.725636:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574264.725647:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574264.725649:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574264.725654:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574264.725659:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574264.725668:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11195
00000400:00000200:1.0:1788574264.725679:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574264.725686:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574264.725689:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574264.725693:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725696:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574264.725698:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725701:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574264.725705:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574264.725707:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574264.725709:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574264.725712:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574264.725715:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574264.725717:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574264.725720:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725722:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574264.725724:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725727:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574264.725730:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574264.725733:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725736:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574264.725739:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574264.725742:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574264.725745:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574264.725747:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574264.725750:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574264.725756:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788574264.725759:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574264.725763:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574264.725765:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574264.725768:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574264.725771:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574264.725775:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574264.725781:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788574264.725788:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574264.725798:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393121408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574264.725809:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574264.725813:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574264.725817:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393121408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574264.725838:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574264.725842:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574264.725845:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574264.725847:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574264.725849:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574264.725852:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574264.725855:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11195
00000400:00000200:1.0:1788574264.725860:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574264.725864:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574264.725867:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574264.725870:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725873:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574264.725879:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725884:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574264.725890:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574264.725892:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574264.725896:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574264.725900:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574264.725905:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574264.725909:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574264.725918:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725924:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574264.725930:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725935:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574264.725940:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574264.725948:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574264.725952:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574264.725956:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574264.725963:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574264.725968:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574264.725972:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574264.725977:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574264.725982:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574264.725986:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574264.725995:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788574264.726001:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574264.726008:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574264.726011:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574264.726016:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574264.726019:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574264.726022:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574264.726031:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574264.726039:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574264.726049:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393121536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574264.726065:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574264.726068:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574264.726075:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393121536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574264.726096:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574264.726111:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574264.726120:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574264.726125:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574264.726129:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574264.726137:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574264.726142:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11195
00000400:00000200:1.0:1788574264.726151:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574264.726158:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574264.726164:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:2.1:1788574264.726170:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788574264.726172:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.726177:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788574264.726177:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:1.0:1788574264.726182:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.726186:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574264.726190:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574264.726194:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574264.726199:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574264.726202:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574264.726205:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574264.726211:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574264.726219:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.726223:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574264.726229:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574264.726233:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574264.726239:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574264.726247:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574264.726253:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574264.726256:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574264.726262:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574264.726268:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574264.726273:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574264.726277:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574264.726283:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574264.726287:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574264.726295:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788574264.726301:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574264.726307:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574264.726310:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574264.726313:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574264.726317:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574264.726319:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574264.726326:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574264.726330:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574264.726341:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393121664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574264.726354:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574264.726359:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574264.726367:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393121664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574264.726388:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574264.726462:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574264.726470:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574264.726480:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11205)
00000800:00000200:2.1:1788574264.726707:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574264.726708:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788574264.726923:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574264.726925:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:1.0:1788574265.685472:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305869241, at 4305869248
00000100:00000010:1.0:1788574265.685496:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000100:00000040:0.0:1788574265.685512:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305869241, at 4305869248
00000100:00000010:0.0:1788574265.685526:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000020:00000001:0.0:1788574266.133612:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574266.133619:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574266.133624:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574267.157602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574267.157605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574267.157608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574269.205556:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574269.205560:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574269.205614:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788574269.845511:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574269.845520:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574269.845522:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574269.845526:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574269.845531:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574269.845538:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11200
00000400:00000200:1.0:1788574269.845548:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574269.845555:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574269.845559:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574269.845563:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845567:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574269.845569:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845572:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574269.845576:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574269.845578:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574269.845581:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574269.845584:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574269.845587:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574269.845590:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574269.845593:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845595:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574269.845598:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845600:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574269.845603:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574269.845607:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845610:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574269.845613:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574269.845617:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574269.845620:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574269.845622:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574269.845624:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574269.845633:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788574269.845635:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574269.845638:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574269.845640:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574269.845642:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574269.845644:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574269.845648:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574269.845654:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574269.845660:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574269.845668:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393121792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574269.845676:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574269.845679:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574269.845683:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393121792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574269.845701:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574269.845704:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574269.845707:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574269.845708:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574269.845709:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574269.845711:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574269.845713:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11200
00000400:00000200:1.0:1788574269.845717:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574269.845719:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574269.845721:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574269.845723:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845724:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574269.845726:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845727:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574269.845729:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574269.845730:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574269.845732:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574269.845733:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574269.845735:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574269.845737:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574269.845740:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845741:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574269.845742:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845744:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574269.845746:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574269.845748:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845750:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574269.845754:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574269.845758:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574269.845763:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574269.845765:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574269.845771:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574269.845776:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574269.845781:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574269.845786:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788574269.845791:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574269.845797:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574269.845799:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574269.845802:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574269.845805:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574269.845808:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574269.845814:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788574269.845820:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574269.845829:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393121920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574269.845840:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574269.845845:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574269.845850:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393121920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574269.845864:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574269.845866:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574269.845870:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574269.845874:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574269.845875:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574269.845878:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574269.845883:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11200
00000400:00000200:1.0:1788574269.845890:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574269.845893:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574269.845897:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574269.845901:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845906:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574269.845909:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845914:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574269.845918:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574269.845921:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574269.845923:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574269.845927:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574269.845929:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574269.845933:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574269.845940:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845943:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574269.845946:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845948:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574269.845950:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574269.845955:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574269.845958:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574269.845959:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574269.845963:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574269.845967:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574269.845969:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574269.845971:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574269.845980:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574269.845984:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574269.845992:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788574269.845999:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574269.846006:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574269.846010:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574269.846016:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574269.846021:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574269.846025:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574269.846029:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574269.846035:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574269.846045:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393122048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574269.846054:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574269.846057:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574269.846063:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393122048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788574269.846076:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574269.846082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:1.0:1788574269.846083:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574269.846090:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574269.846097:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574269.846107:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11211)
00000800:00000200:2.1:1788574269.846368:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574269.846370:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788574269.846602:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574269.846604:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:1.0:1788574270.229616:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574270.229619:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574270.229622:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574270.805466:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305874361, at 4305874368
00000100:00000040:1.0:1788574270.805466:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305874361, at 4305874368
00000100:00000010:1.0:1788574270.805494:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000100:00000010:0.0:1788574270.805495:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000020:00000001:0.0:1788574272.277605:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574272.277608:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574272.277611:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574273.301569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574273.301574:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574273.301577:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788574274.965506:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574274.965516:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574274.965520:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574274.965524:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574274.965529:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574274.965537:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11206
00000400:00000200:1.0:1788574274.965546:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574274.965552:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574274.965555:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574274.965559:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965562:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574274.965565:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965568:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574274.965572:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574274.965574:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574274.965577:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574274.965580:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574274.965584:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574274.965586:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574274.965590:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965593:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574274.965596:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965598:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574274.965602:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574274.965607:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965610:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574274.965613:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574274.965617:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574274.965620:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574274.965623:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574274.965625:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574274.965634:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788574274.965637:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574274.965642:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574274.965643:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574274.965646:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574274.965649:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574274.965653:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574274.965661:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574274.965668:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574274.965679:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393122176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574274.965690:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574274.965695:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574274.965700:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393122176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574274.965724:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574274.965728:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574274.965731:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574274.965733:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574274.965735:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574274.965739:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574274.965743:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11206
00000400:00000200:1.0:1788574274.965752:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574274.965759:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574274.965763:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574274.965772:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965778:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574274.965784:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965788:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574274.965794:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574274.965797:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574274.965802:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574274.965806:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574274.965812:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574274.965818:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574274.965825:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965831:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574274.965836:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965840:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574274.965846:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574274.965852:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574274.965857:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574274.965860:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574274.965866:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574274.965872:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574274.965877:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574274.965883:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574274.965887:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574274.965893:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574274.965900:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788574274.965902:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574274.965909:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574274.965912:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574274.965915:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574274.965919:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574274.965924:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574274.965932:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788574274.965938:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574274.965948:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393122304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574274.965965:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574274.965969:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574274.965977:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393122304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574274.966012:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574274.966016:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574274.966020:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574274.966024:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574274.966028:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574274.966035:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574274.966042:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11206
00000400:00000200:1.0:1788574274.966048:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574274.966052:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574274.966055:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574274.966062:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.966067:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574274.966074:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.1:1788574274.966079:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574274.966080:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.1:1788574274.966082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:1.0:1788574274.966085:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574274.966089:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574274.966094:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574274.966098:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574274.966102:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574274.966109:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574274.966116:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.966120:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574274.966127:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574274.966133:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574274.966139:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574274.966145:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574274.966149:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574274.966152:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574274.966156:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574274.966171:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574274.966177:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574274.966181:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574274.966187:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574274.966191:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574274.966196:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788574274.966203:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574274.966209:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574274.966213:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574274.966218:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574274.966220:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574274.966222:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574274.966230:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788574274.966236:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574274.966247:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393122432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574274.966268:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574274.966272:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574274.966290:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393122432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574274.966325:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574274.966331:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574274.966338:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574274.966348:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11216)
00000800:00000200:2.1:1788574274.966584:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574274.966586:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788574274.966782:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574274.966784:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:0.0:1788574275.349611:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574275.349614:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574275.349617:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574275.925491:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305879481, at 4305879488
00000100:00000040:0.0:1788574275.925497:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305879481, at 4305879488
00000100:00000010:1.0:1788574275.925507:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000100:00000010:0.0:1788574275.925509:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000020:00000001:1.0:1788574276.373647:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574276.373654:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574276.373657:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574278.421627:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574278.421631:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574278.421634:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574279.445614:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574279.445619:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574279.445622:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574280.085518:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574280.085530:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574280.085532:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574280.085536:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574280.085540:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574280.085547:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11211
00000400:00000200:0.0:1788574280.085556:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574280.085562:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574280.085566:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574280.085569:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085572:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574280.085575:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085578:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574280.085582:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574280.085583:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574280.085586:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574280.085589:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574280.085592:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574280.085595:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574280.085598:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085600:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574280.085602:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085604:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574280.085608:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574280.085611:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085614:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574280.085617:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574280.085620:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574280.085623:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574280.085625:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574280.085628:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574280.085637:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788574280.085640:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574280.085644:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574280.085646:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574280.085649:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574280.085652:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574280.085656:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574280.085663:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574280.085670:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574280.085678:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393122560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574280.085688:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574280.085692:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574280.085697:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393122560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574280.085721:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574280.085725:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574280.085729:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574280.085731:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574280.085733:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574280.085736:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574280.085739:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11211
00000400:00000200:0.0:1788574280.085743:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574280.085747:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574280.085750:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574280.085752:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085755:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574280.085757:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085759:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574280.085762:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574280.085763:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574280.085765:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574280.085768:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574280.085770:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574280.085772:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574280.085776:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085778:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574280.085781:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085784:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574280.085787:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574280.085790:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574280.085793:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574280.085796:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574280.085803:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574280.085808:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574280.085812:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574280.085818:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574280.085823:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574280.085829:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574280.085836:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788574280.085842:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574280.085849:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574280.085852:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574280.085856:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574280.085860:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574280.085871:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574280.085882:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574280.085888:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574280.085898:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393122688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574280.085915:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574280.085920:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574280.085931:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393122688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574280.085951:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574280.085957:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574280.085962:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574280.085963:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574280.085967:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574280.085971:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574280.085977:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11211
00000400:00000200:0.0:1788574280.085986:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574280.085995:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574280.086002:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574280.086011:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.086017:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574280.086022:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.086025:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574280.086028:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574280.086031:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574280.086036:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574280.086041:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574280.086046:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574280.086051:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574280.086055:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.086059:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574280.086064:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574280.086070:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574280.086076:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574280.086083:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:2.1:1788574280.086088:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574280.086090:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574280.086092:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
02000000:00000001:0.0:1788574280.086094:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574280.086100:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574280.086106:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574280.086110:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574280.086115:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574280.086129:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574280.086134:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574280.086142:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788574280.086148:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574280.086155:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574280.086159:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574280.086163:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574280.086167:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574280.086170:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574280.086179:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574280.086186:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574280.086196:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393122816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574280.086208:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574280.086213:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574280.086223:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393122816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574280.086238:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574280.086244:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574280.086253:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574280.086259:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11221)
00000800:00000200:2.1:1788574280.086622:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574280.086625:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574280.086888:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574280.086890:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:1.0:1788574281.045489:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305884601, at 4305884608
00000100:00000040:0.0:1788574281.045491:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305884601, at 4305884608
00000100:00000010:1.0:1788574281.045504:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000100:00000010:0.0:1788574281.045507:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000020:00000001:0.0:1788574281.493610:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574281.493615:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574281.493618:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574282.517604:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574282.517608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574282.517612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574284.565550:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574284.565554:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574284.565556:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574285.206312:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574285.206320:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574285.206966:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574285.206968:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574285.207222:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574285.207224:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1788574285.589574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574285.589578:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574285.589580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574286.165487:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305889722, at 4305889728
00000100:00000040:0.0:1788574286.165493:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305889722, at 4305889728
00000100:00000010:1.0:1788574286.165500:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000100:00000010:0.0:1788574286.165503:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000020:00000001:0.0:1788574287.637583:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574287.637588:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574287.637591:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574288.661539:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574288.661544:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574288.661548:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788574290.325465:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574290.325475:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574290.325481:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574290.325631:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574290.325642:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574290.325650:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574290.325657:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574290.325664:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788574290.326079:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574290.326083:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574290.326543:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574290.326546:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574290.326803:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574290.326806:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1788574290.709586:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574290.709591:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574290.709595:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574291.285475:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305894841, at 4305894848
00000100:00000040:1.0:1788574291.285475:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305894841, at 4305894848
00000100:00000010:1.0:1788574291.285490:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00000100:00000010:0.0:1788574291.285496:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000020:00000001:1.0:1788574291.733604:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574291.733608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574291.733612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574293.781527:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574293.781530:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574293.781533:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574294.805602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574294.805605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574294.805608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574295.445975:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574295.445981:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788574295.446434:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574295.446436:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788574295.446631:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574295.446633:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:0.0:1788574296.405501:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305899961, at 4305899968
00000100:00000040:1.0:1788574296.405501:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305899961, at 4305899968
00000100:00000010:0.0:1788574296.405516:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000100:00000010:1.0:1788574296.405517:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00000020:00000001:0.0:1788574296.853569:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574296.853573:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574296.853576:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574297.877576:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574297.877580:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574297.877585:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574299.925542:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574299.925545:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574299.925547:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574300.566031:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574300.566036:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574300.566555:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574300.566556:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574300.566744:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574300.566746:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:1.0:1788574300.949583:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574300.949587:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574300.949591:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574301.525471:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305905081, at 4305905088
00000100:00000040:0.0:1788574301.525479:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305905081, at 4305905088
00000100:00000010:1.0:1788574301.525486:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000100:00000010:0.0:1788574301.525489:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000020:00000001:0.0:1788574302.997585:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574302.997587:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574302.997590:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574304.021601:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574304.021605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574304.021609:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574305.686038:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574305.686046:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574305.686416:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574305.686418:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574305.686609:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574305.686611:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:0.0:1788574306.069564:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574306.069568:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574306.069571:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574306.645497:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305910201, at 4305910208
00000100:00000040:1.0:1788574306.645497:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305910201, at 4305910208
00000100:00000010:0.0:1788574306.645512:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00000100:00000010:1.0:1788574306.645514:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000020:00000001:1.0:1788574307.093569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574307.093573:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574307.093577:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574309.141573:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574309.141577:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574309.141580:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788574310.165606:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574310.165612:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574310.165616:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574310.806117:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574310.806349:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574310.806517:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574311.317549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574311.317556:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574311.317559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574311.317565:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574311.317568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574311.317572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574311.317575:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574311.317578:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574311.317581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574311.765488:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305915321, at 4305915328
00000100:00000040:0.0:1788574311.765489:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305915321, at 4305915328
00000100:00000010:1.0:1788574311.765506:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000100:00000010:0.0:1788574311.765508:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000020:00000001:0.0:1788574312.213645:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574312.213651:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574312.213655:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574312.341493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574312.341499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574312.341501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574312.341505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574312.341508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574312.341511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574312.341513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574312.341515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574312.341517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574313.237641:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574313.237645:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574313.237648:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574313.365445:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574313.365451:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574313.365453:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574313.365457:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574313.365460:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574313.365463:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574313.365465:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574313.365468:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574313.365470:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574313.877924:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574313.877931:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574313.877936:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574313.877949:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574310/real 1788574313]  req@ffff89536d1e2680 x1875444393124992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574335 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574313.877972:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574313.877976:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574313.877981:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f478.
00000400:00000200:0.0:1788574313.877988:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:0.0:1788574313.877993:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:0.0:1788574313.877999:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574313.878007:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393124992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574313.878022:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393124992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878034:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574313.878038:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574313.878048:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574313.878058:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393124992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878066:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574313.878072:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393124992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878079:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574313.878082:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878084:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574313.878085:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878087:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574313.878091:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574313.878095:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574313.878100:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574313.878102:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574313.878106:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393124992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574313.878115:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393124992:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574313.878120:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574313.878123:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574310/real 1788574313]  req@ffff89536d1e2a00 x1875444393125120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574313.878132:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574313.878133:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574313.878136:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fe18.
00000400:00000200:0.0:1788574313.878139:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c653b8
00000400:00000010:0.0:1788574313.878140:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c653b8.
00000100:00000001:0.0:1788574313.878143:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574313.878146:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393125120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574313.878154:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393125120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878174:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574313.878175:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878177:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574313.878179:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574313.878183:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393125120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878190:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574313.878194:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393125120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878201:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574313.878204:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878205:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574313.878207:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878209:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574313.878247:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574313.878250:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574313.878254:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574313.878257:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574313.878262:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393125120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574313.878270:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393125120:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574313.878274:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574313.878279:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574310/real 1788574313]  req@ffff89536d1e2d80 x1875444393124864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574313.878286:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574313.878287:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574313.878289:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fe70.
00000400:00000200:0.0:1788574313.878292:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:0.0:1788574313.878294:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:0.0:1788574313.878296:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574313.878300:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393124864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574313.878308:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393124864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878314:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574313.878315:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878317:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574313.878318:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574313.878321:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393124864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878327:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574313.878331:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393124864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878337:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574313.878339:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878341:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574313.878342:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878344:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574313.878346:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574313.878349:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574313.878352:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574313.878354:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574313.878357:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393124864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574313.878364:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393124864:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574313.878369:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574313.878380:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393124992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878386:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574313.878390:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574313.878448:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788574313.878454:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574313.878458:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574313.878461:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574313.878464:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574313.878468:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:0.0:1788574313.878472:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574313.878474:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574313.878478:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:0.0:1788574313.878483:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574313.878485:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878487:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574313.878493:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393125120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878501:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574313.878503:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574313.878505:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788574313.878509:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574313.878510:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574313.878512:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574313.878515:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574313.878517:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788574313.878519:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574313.878520:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574313.878522:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:0.0:1788574313.878524:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574313.878526:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878527:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574313.878531:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393124864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574335 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574313.878538:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574313.878539:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574313.878541:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574313.878544:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574313.878545:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574313.878546:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574313.878553:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574313.878554:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788574313.878558:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574313.878559:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574313.878560:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788574313.878563:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574313.878564:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878567:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574313.878577:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574313.878580:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574313.878583:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574313.878585:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574314.389495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574314.389503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574314.389505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574314.389511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574314.389515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574314.389528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574314.389531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574314.389537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574314.389540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574315.285560:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574315.285564:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574315.285568:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574315.413475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574315.413482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574315.413484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574315.413490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574315.413494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574315.413497:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574315.413500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574315.413502:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574315.413505:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574315.925774:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574315.925794:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574315.925799:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574315.925812:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.925832:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574315.925837:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574315.925844:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574315.925850:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574315.925857:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574315.925866:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393125248:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574315.925877:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574315.925887:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574315.925892:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574315.925900:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574315.925910:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574315.925917:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574315.925931:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574315.925939:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574315.925951:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574315.925967:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393125248, portal 25
00000100:00000001:0.0:1788574315.925973:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574315.925977:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:0.0:1788574315.925991:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926014:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574315.926023:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574315.926030:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574315.926037:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393125248, offset 0
00000400:00000200:0.0:1788574315.926049:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574315.926063:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574315.926079:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574315.926090:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574315.926098:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574315.926106:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574315.926114:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574315.926125:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330234:330234:256:4294967295] 192.168.202.110@tcp LPNI seq info [330191:330191:8:4294967295]
00000400:00000200:0.0:1788574315.926136:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574315.926152:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574315.926162:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574315.926173:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574315.926187:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574315.926195:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574315.926207:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574315.926215:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574315.926224:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574315.926263:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574315.926274:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574315.926279:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574315.926286:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574315.926288:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574315.926292:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574315.926297:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926308:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574315.926312:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926319:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574315.926321:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574315.926327:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574315.926331:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574315.926334:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305920441
00000100:00000010:0.0:1788574315.926340:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574315.926347:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574315.926353:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305920441
00000100:00000010:0.0:1788574315.926356:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ba0.
00000100:00000001:0.0:1788574315.926358:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574315.926361:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574315.926365:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574315.926367:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574315.926371:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574315/real 1788574315]  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574315.926388:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574315.926390:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574315.926436:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574315.926440:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574315.926442:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574315.926445:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574315.926459:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574315.926470:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926476:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574315.926478:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.926481:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574315.926483:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574315.926488:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926496:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574315.926502:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926510:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574315.926514:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.926516:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574315.926517:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.926520:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574315.926522:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574315.926526:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574315.926530:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574315.926533:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574315.926539:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574315.926547:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393125248:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574315.926553:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.926555:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574315.926559:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393125248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926565:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574315.926567:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574315.926570:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574315.926574:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574315.926576:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574315.926578:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574315.926580:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574315.926583:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788574315.926587:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574315.926589:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574315.926591:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:0.0:1788574315.926594:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574315.926596:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.926598:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.926606:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574315.926609:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574315.926611:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574315.926612:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574315.926614:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574315.926618:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926625:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574315.926627:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574315.926630:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574315.926632:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574315.926635:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574315.926638:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393125376:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574315.926641:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574315.926644:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574315.926646:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574315.926648:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574315.926651:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574315.926668:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574315.926673:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574315.926676:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574315.926681:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574315.926684:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393125376, portal 10
00000100:00000001:0.0:1788574315.926686:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574315.926688:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000040:0.0:1788574315.926695:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926703:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574315.926706:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574315.926708:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574315.926710:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393125376, offset 0
00000400:00000200:0.0:1788574315.926713:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574315.926717:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574315.926721:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574315.926723:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574315.926725:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574315.926727:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574315.926728:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574315.926731:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330235:330235:256:4294967295] 192.168.202.110@tcp LPNI seq info [330192:330192:8:4294967295]
00000400:00000200:0.0:1788574315.926734:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574315.926739:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574315.926743:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574315.926747:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574315.926751:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574315.926754:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574315.926758:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574315.926761:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574315.926765:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574315.926768:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574315.926771:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574315.926774:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574315.926777:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574315.926779:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574315.926781:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574315.926785:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926794:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574315.926797:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926803:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574315.926804:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574315.926808:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574315.926812:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574315.926816:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574315.926818:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305919527
00000100:00000001:0.0:1788574315.926821:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305920441
00000100:00000001:0.0:1788574315.926823:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305919527
00000100:00000001:0.0:1788574315.926825:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305920441
00000100:00000001:0.0:1788574315.926827:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574315.926829:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574315.926831:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574315.926835:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574315/real 1788574315]  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574315.926851:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574315.926852:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574315.926854:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574315.926857:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574315.926858:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574315.926861:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574315.926864:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574315.926872:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926877:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574315.926879:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.926880:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574315.926882:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574315.926886:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926893:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574315.926897:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926904:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574315.926906:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.926908:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574315.926909:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.926911:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574315.926913:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574315.926916:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574315.926920:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574315.926922:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574315.926927:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574315.926936:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393125376:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574315.926940:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574315.926944:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.926951:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574315.926953:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574315.926955:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574315.926957:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574315.926960:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574315.926963:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393125504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574315.926967:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574315.926969:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574315.926971:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574315.926972:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574315.926977:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574315.926978:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574315.926982:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574315.926985:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574315.926988:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574315.926991:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393125504, portal 4
00000100:00000001:0.0:1788574315.926993:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574315.926995:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:0.0:1788574315.927003:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.927010:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574315.927013:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574315.927016:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574315.927018:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393125504, offset 0
00000400:00000200:0.0:1788574315.927024:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574315.927028:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574315.927033:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574315.927037:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574315.927040:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574315.927043:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574315.927045:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574315.927049:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330236:330236:256:4294967295] 192.168.202.110@tcp LPNI seq info [330193:330193:8:4294967295]
00000400:00000200:0.0:1788574315.927053:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574315.927061:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574315.927067:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574315.927070:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574315.927076:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574315.927079:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574315.927084:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574315.927087:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574315.927092:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574315.927094:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574315.927097:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574315.927100:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574315.927103:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574315.927104:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574315.927107:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574315.927110:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.927117:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574315.927120:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.927126:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574315.927128:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574315.927130:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574315.927133:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574315.927136:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574315.927139:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305919673
00000100:00000001:0.0:1788574315.927141:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305920441
00000100:00000001:0.0:1788574315.927143:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305919673
00000100:00000001:0.0:1788574315.927145:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305920441
00000100:00000001:0.0:1788574315.927147:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574315.927158:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574315.927160:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574315.927165:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574315/real 1788574315]  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574315.927173:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574315.927175:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574315.927177:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574315.927179:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574315.927181:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574315.927184:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574315.927187:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574315.927195:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.927202:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574315.927203:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.927205:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574315.927206:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574315.927210:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.927217:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574315.927221:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.927228:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574315.927244:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.927246:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574315.927247:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.927249:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574315.927251:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574315.927254:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574315.927258:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574315.927260:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574315.927265:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574315.927273:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393125504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574315.927276:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.927278:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574315.927281:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393125376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.927288:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574315.927290:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574315.927293:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574315.927296:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574315.927298:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574315.927299:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574315.927302:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574315.927304:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788574315.927307:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574315.927308:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574315.927310:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:0.0:1788574315.927313:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574315.927314:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.927315:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574315.927318:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393125504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574340 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574315.927326:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574315.927327:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574315.927333:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574315.927335:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574315.927336:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574315.927337:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574315.927338:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574315.927340:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788574315.927341:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574315.927342:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574315.927343:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:0.0:1788574315.927344:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574315.927345:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.927346:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574315.927350:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574315.927351:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574315.927353:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574315.927355:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574316.309640:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574316.309646:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574316.309652:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574316.437462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574316.437469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574316.437471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574316.437475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574316.437478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574316.437481:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574316.437482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574316.437484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574316.437487:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574316.885491:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305920441, at 4305920448
00000100:00000040:1.0:1788574316.885491:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305920441, at 4305920448
00000100:00000010:1.0:1788574316.885507:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ba0.
00000100:00000010:0.0:1788574316.885509:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:1.0:1788574317.461517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574317.461524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574317.461526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574317.461531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574317.461534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574317.461538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574317.461540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574317.461542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574317.461545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574318.357640:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574318.357644:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574318.357647:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574318.485505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574318.485516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574318.485520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574318.485528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574318.485534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574318.485539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574318.485543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574318.485547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574318.485551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574319.381583:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574319.381589:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574319.381593:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574319.509532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574319.509540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574319.509542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574319.509547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574319.509551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574319.509555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574319.509558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574319.509561:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574319.509564:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574320.533563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574320.533571:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574320.533573:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574320.533580:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574320.533585:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574320.533590:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574320.533593:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574320.533597:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574320.533601:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574321.045808:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574321.045817:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574321.045827:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574321.045844:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.045881:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574321.045887:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574321.045896:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574321.045904:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574321.045914:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574321.045929:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393125632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574321.045946:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574321.045955:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574321.045964:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574321.045975:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574321.045989:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574321.045997:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574321.046016:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574321.046024:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574321.046039:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574321.046050:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393125632, portal 25
00000100:00000001:0.0:1788574321.046061:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574321.046067:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:0.0:1788574321.046085:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046093:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574321.046102:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574321.046105:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574321.046113:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393125632, offset 0
00000400:00000200:0.0:1788574321.046131:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574321.046142:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574321.046150:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574321.046155:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574321.046159:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574321.046163:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574321.046165:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574321.046169:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330237:330237:256:4294967295] 192.168.202.110@tcp LPNI seq info [330194:330194:8:4294967295]
00000400:00000200:0.0:1788574321.046173:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574321.046179:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574321.046184:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574321.046189:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574321.046194:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574321.046198:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574321.046203:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574321.046206:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574321.046209:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574321.046211:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574321.046221:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574321.046225:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574321.046229:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574321.046231:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574321.046234:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574321.046237:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046245:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574321.046248:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574346 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046253:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574321.046254:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574321.046258:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574321.046261:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574321.046263:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305925561
00000100:00000010:0.0:1788574321.046268:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574321.046273:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574321.046303:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305925561
00000100:00000010:0.0:1788574321.046306:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d76c0.
00000100:00000001:0.0:1788574321.046309:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574321.046311:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574321.046313:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574321.046315:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574321.046319:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574321/real 1788574321]  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574321.046327:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574321.046328:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574321.046330:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574321.046332:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574321.046334:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574321.046335:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574321.046338:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574321.046343:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046347:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574321.046349:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046350:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574321.046352:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574321.046360:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046365:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574321.046369:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046374:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574321.046375:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046376:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574321.046377:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046379:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574321.046381:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574321.046383:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574321.046386:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574321.046388:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574321.046433:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574321.046444:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393125632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574321.046452:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046455:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574321.046459:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393125632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046466:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574321.046469:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574321.046472:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574321.046476:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574321.046479:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574321.046481:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574321.046483:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574321.046486:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788574321.046490:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574321.046492:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574321.046494:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:0.0:1788574321.046498:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574321.046499:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046502:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046510:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574321.046512:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574321.046515:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574321.046516:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574321.046518:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574321.046523:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046531:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574321.046532:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574321.046535:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574321.046537:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574321.046539:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574321.046543:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393125760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574321.046546:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574321.046549:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574321.046551:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574321.046553:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574321.046557:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574321.046559:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574321.046563:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574321.046566:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574321.046570:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574321.046573:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393125760, portal 10
00000100:00000001:0.0:1788574321.046575:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574321.046576:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:0.0:1788574321.046584:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046601:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574321.046603:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574321.046606:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574321.046609:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393125760, offset 0
00000400:00000200:0.0:1788574321.046614:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574321.046618:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574321.046622:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574321.046626:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574321.046628:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574321.046631:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574321.046634:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574321.046637:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330238:330238:256:4294967295] 192.168.202.110@tcp LPNI seq info [330195:330195:8:4294967295]
00000400:00000200:0.0:1788574321.046642:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574321.046648:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574321.046654:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574321.046657:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574321.046661:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574321.046664:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574321.046668:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574321.046671:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574321.046673:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574321.046675:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574321.046678:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574321.046680:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574321.046683:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574321.046684:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574321.046686:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574321.046689:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046694:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574321.046696:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574346 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046701:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574321.046702:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574321.046704:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574321.046707:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574321.046709:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574321.046711:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305924647
00000100:00000001:0.0:1788574321.046713:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305925561
00000100:00000001:0.0:1788574321.046714:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305924647
00000100:00000001:0.0:1788574321.046716:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305925561
00000100:00000001:0.0:1788574321.046717:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574321.046718:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574321.046720:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574321.046722:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574321/real 1788574321]  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574321.046728:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574321.046728:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574321.046730:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574321.046731:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574321.046732:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574321.046734:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574321.046736:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574321.046741:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046746:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574321.046746:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046748:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574321.046748:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574321.046751:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046755:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574321.046757:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046761:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574321.046763:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046764:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574321.046764:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046765:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574321.046767:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574321.046769:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574321.046772:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574321.046773:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574321.046776:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574321.046781:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393125760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574321.046783:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574321.046786:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046795:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574321.046796:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574321.046797:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574321.046799:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574321.046800:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574321.046802:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393125888:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574321.046804:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574321.046805:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574321.046807:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574321.046808:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574321.046812:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574321.046813:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574321.046815:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574321.046817:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574321.046820:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574321.046822:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393125888, portal 4
00000100:00000001:0.0:1788574321.046823:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574321.046824:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:0.0:1788574321.046828:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046832:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574321.046834:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574321.046835:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574321.046836:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393125888, offset 0
00000400:00000200:0.0:1788574321.046839:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574321.046841:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574321.046843:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574321.046845:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574321.046847:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574321.046848:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574321.046850:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574321.046852:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330239:330239:256:4294967295] 192.168.202.110@tcp LPNI seq info [330196:330196:8:4294967295]
00000400:00000200:0.0:1788574321.046855:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574321.046861:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574321.046866:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574321.046869:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574321.046873:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574321.046876:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574321.046880:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574321.046883:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574321.046887:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574321.046889:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574321.046892:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574321.046896:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574321.046899:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574321.046901:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574321.046903:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574321.046907:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046916:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574321.046918:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574346 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046923:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574321.046924:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574321.046927:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574321.046929:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574321.046932:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574321.046933:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305924793
00000100:00000001:0.0:1788574321.046935:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305925561
00000100:00000001:0.0:1788574321.046937:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305924793
00000100:00000001:0.0:1788574321.046938:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305925561
00000100:00000001:0.0:1788574321.046940:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574321.046941:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574321.046943:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574321.046945:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574321/real 1788574321]  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574321.046951:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574321.046951:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574321.046952:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574321.046954:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574321.046955:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574321.046956:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574321.046958:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574321.046964:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046967:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574321.046968:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046969:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574321.046970:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574321.046972:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046976:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574321.046979:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.046983:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574321.046984:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046985:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574321.046986:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.046987:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574321.046989:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574321.046991:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574321.046993:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574321.046995:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574321.046997:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574321.047002:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393125888:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574321.047004:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.047006:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574321.047007:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393125760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.047011:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574321.047012:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574321.047014:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574321.047015:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574321.047016:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574321.047017:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574321.047018:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574321.047019:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788574321.047021:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574321.047022:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574321.047023:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:0.0:1788574321.047025:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574321.047026:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.047027:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574321.047029:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393125888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574346 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574321.047033:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574321.047034:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574321.047035:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574321.047036:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574321.047037:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574321.047037:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574321.047039:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574321.047040:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574321.047041:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574321.047042:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574321.047043:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:0.0:1788574321.047045:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574321.047046:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.047047:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574321.047049:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574321.047050:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574321.047051:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574321.047053:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574321.429604:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574321.429608:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574321.429611:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574321.557556:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574321.557566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574321.557568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574321.557574:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574321.557577:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574321.557582:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574321.557585:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574321.557588:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574321.557590:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574322.005551:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305925561, at 4305925568
00000100:00000040:0.0:1788574322.005552:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305925561, at 4305925568
00000100:00000010:1.0:1788574322.005574:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d76c0.
00000100:00000010:0.0:1788574322.005583:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788574322.453602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574322.453607:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574322.453611:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574322.581517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574322.581526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574322.581529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574322.581534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574322.581538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574322.581542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574322.581545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574322.581548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574322.581550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574323.605514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574323.605523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574323.605525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574323.605532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574323.605536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574323.605545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574323.605548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574323.605551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574323.605554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574324.501610:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574324.501615:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574324.501619:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574324.629483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574324.629490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574324.629492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574324.629497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574324.629499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574324.629503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574324.629505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574324.629507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574324.629509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574325.525551:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574325.525555:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574325.525559:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574325.653509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574325.653516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574325.653519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574325.653524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574325.653528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574325.653532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574325.653535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574325.653546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574325.653551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574326.165773:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574326.165785:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574326.165792:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574326.165816:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.165834:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574326.165842:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574326.165860:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574326.165867:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574326.165875:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574326.165886:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393126016:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574326.165892:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574326.165902:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574326.165908:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574326.165912:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574326.165921:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574326.165927:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574326.165943:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574326.165948:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574326.165959:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574326.165970:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393126016, portal 25
00000100:00000001:0.0:1788574326.165977:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574326.165983:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:0.0:1788574326.165999:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166026:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574326.166040:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574326.166044:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574326.166054:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393126016, offset 0
00000400:00000200:0.0:1788574326.166073:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574326.166097:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574326.166114:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574326.166131:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574326.166148:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574326.166163:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574326.166175:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574326.166186:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330240:330240:256:4294967295] 192.168.202.110@tcp LPNI seq info [330197:330197:8:4294967295]
00000400:00000200:0.0:1788574326.166200:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574326.166231:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574326.166248:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574326.166255:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574326.166276:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574326.166281:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574326.166298:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574326.166305:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574326.166314:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574326.166318:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574326.166326:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574326.166346:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574326.166357:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574326.166362:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574326.166375:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574326.166383:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166445:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574326.166449:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574351 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166456:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574326.166460:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574326.166465:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574326.166468:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574326.166470:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305930682
00000100:00000010:0.0:1788574326.166475:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574326.166480:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574326.166485:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305930682
00000100:00000010:0.0:1788574326.166487:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7de0.
00000100:00000001:0.0:1788574326.166489:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574326.166492:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574326.166493:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574326.166495:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574326.166498:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574326/real 1788574326]  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574326.166506:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574326.166508:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574326.166509:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574326.166512:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574326.166513:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574326.166514:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574326.166517:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574326.166522:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166527:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574326.166529:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.166531:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574326.166533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574326.166537:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166545:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574326.166550:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166558:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574326.166561:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.166562:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574326.166564:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.166567:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574326.166569:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574326.166573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574326.166578:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574326.166581:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574326.166586:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574326.166597:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393126016:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574326.166610:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.166613:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574326.166617:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393126016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166625:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574326.166627:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574326.166631:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574326.166634:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574326.166637:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574326.166640:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574326.166642:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574326.166645:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788574326.166649:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574326.166651:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574326.166653:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:0.0:1788574326.166657:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574326.166659:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.166661:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.166668:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574326.166670:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574326.166673:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574326.166675:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574326.166677:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574326.166682:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166691:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574326.166692:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574326.166695:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574326.166697:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574326.166700:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574326.166704:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126144:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574326.166708:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574326.166710:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574326.166712:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574326.166714:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574326.166718:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574326.166720:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574326.166725:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574326.166728:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574326.166732:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574326.166736:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393126144, portal 10
00000100:00000001:0.0:1788574326.166738:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574326.166740:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:0.0:1788574326.166747:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166755:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574326.166757:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574326.166760:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574326.166763:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393126144, offset 0
00000400:00000200:0.0:1788574326.166767:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574326.166772:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574326.166777:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574326.166780:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574326.166783:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574326.166786:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574326.166789:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574326.166792:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330241:330241:256:4294967295] 192.168.202.110@tcp LPNI seq info [330198:330198:8:4294967295]
00000400:00000200:0.0:1788574326.166797:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574326.166805:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574326.166810:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574326.166814:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574326.166818:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574326.166821:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574326.166826:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574326.166830:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574326.166835:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574326.166837:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574326.166841:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574326.166845:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574326.166848:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574326.166850:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574326.166853:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574326.166856:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166872:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574326.166875:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574351 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166883:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574326.166886:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574326.166889:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574326.166892:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574326.166896:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574326.166898:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305929767
00000100:00000001:0.0:1788574326.166901:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305930682
00000100:00000001:0.0:1788574326.166904:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305929767
00000100:00000001:0.0:1788574326.166906:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305930682
00000100:00000001:0.0:1788574326.166909:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574326.166911:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574326.166913:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574326.166918:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574326/real 1788574326]  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574326.166927:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574326.166929:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574326.166931:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574326.166933:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574326.166935:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574326.166938:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574326.166941:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574326.166951:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166959:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574326.166961:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.166963:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574326.166965:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574326.166969:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166975:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574326.166980:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.166987:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574326.166990:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.166992:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574326.166994:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.166996:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574326.166998:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574326.167002:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574326.167005:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574326.167008:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574326.167013:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574326.167023:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126144:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574326.167027:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574326.167032:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.167040:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574326.167041:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574326.167044:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574326.167046:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574326.167049:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574326.167052:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126272:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574326.167056:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574326.167058:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574326.167060:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574326.167062:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574326.167066:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574326.167068:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574326.167072:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574326.167074:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574326.167078:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574326.167081:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393126272, portal 4
00000100:00000001:0.0:1788574326.167084:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574326.167085:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:0.0:1788574326.167091:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.167099:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574326.167101:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574326.167104:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574326.167106:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393126272, offset 0
00000400:00000200:0.0:1788574326.167111:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574326.167115:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574326.167118:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574326.167121:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574326.167124:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574326.167127:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574326.167129:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574326.167132:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330242:330242:256:4294967295] 192.168.202.110@tcp LPNI seq info [330199:330199:8:4294967295]
00000400:00000200:0.0:1788574326.167137:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574326.167144:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574326.167149:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574326.167153:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574326.167157:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574326.167160:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574326.167165:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574326.167169:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574326.167173:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574326.167176:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574326.167179:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574326.167183:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574326.167186:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574326.167188:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574326.167191:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574326.167195:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.167204:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574326.167207:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574351 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.167215:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574326.167217:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574326.167220:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574326.167223:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574326.167227:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574326.167229:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305929913
00000100:00000001:0.0:1788574326.167232:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305930682
00000100:00000001:0.0:1788574326.167234:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305929913
00000100:00000001:0.0:1788574326.167237:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305930682
00000100:00000001:0.0:1788574326.167239:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574326.167242:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574326.167244:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574326.167248:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574326/real 1788574326]  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574326.167257:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574326.167259:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574326.167261:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574326.167263:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574326.167265:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574326.167267:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574326.167271:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574326.167280:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.167287:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574326.167288:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.167290:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574326.167292:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574326.167296:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.167303:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574326.167307:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.167314:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574326.167317:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.167318:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574326.167319:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.167321:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574326.167323:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574326.167326:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574326.167472:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574326.167474:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574326.167477:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574326.167487:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126272:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574326.167490:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.167492:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574326.167494:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393126144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.167498:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574326.167499:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574326.167501:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574326.167503:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574326.167504:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574326.167505:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574326.167506:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574326.167508:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788574326.167509:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574326.167510:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574326.167511:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:0.0:1788574326.167513:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574326.167514:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.167515:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574326.167517:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393126272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574351 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574326.167521:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574326.167522:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574326.167523:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574326.167524:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574326.167525:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574326.167525:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574326.167527:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574326.167527:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788574326.167529:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574326.167529:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574326.167530:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:0.0:1788574326.167532:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574326.167533:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.167534:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574326.167537:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574326.167538:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574326.167539:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574326.167541:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574326.677490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574326.677497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574326.677500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574326.677505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574326.677509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574326.677513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574326.677516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574326.677519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574326.677522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574327.125468:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305930682, at 4305930688
00000100:00000040:1.0:1788574327.125471:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305930682, at 4305930688
00000100:00000010:0.0:1788574327.125484:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574327.125487:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7de0.
00000020:00000001:0.0:1788574327.573525:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574327.573529:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574327.573533:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574327.701513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574327.701521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574327.701522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574327.701528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574327.701532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574327.701535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574327.701538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574327.701541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574327.701544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574328.597655:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574328.597659:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574328.597662:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574328.725506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574328.725514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574328.725516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574328.725521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574328.725525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574328.725528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574328.725531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574328.725534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574328.725537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574329.237521:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574329.237534:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574329.237540:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574329.237751:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574329.237763:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574329.237771:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574329.237779:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574329.237782:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574329.749520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574329.749533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574329.749536:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574329.749543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574329.749548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574329.749553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574329.749556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574329.749561:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574329.749565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574330.645622:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574330.645626:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574330.645629:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574330.773476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574330.773484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574330.773487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574330.773493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574330.773497:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574330.773501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574330.773504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574330.773506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574330.773509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574331.285749:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574331.285754:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574331.285758:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574331.285767:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.285782:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574331.285785:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574331.285790:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574331.285793:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574331.285797:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574331.285804:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393126400:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574331.285809:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574331.285814:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574331.285818:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574331.285821:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574331.285825:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574331.285827:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574331.285833:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574331.285839:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574331.285847:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574331.285854:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393126400, portal 25
00000100:00000001:0.0:1788574331.285859:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574331.285861:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:0.0:1788574331.285866:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.285878:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574331.285884:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574331.285889:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574331.285893:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393126400, offset 0
00000400:00000200:0.0:1788574331.285900:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574331.285909:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574331.285918:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574331.285923:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574331.285929:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574331.285934:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574331.285939:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574331.285943:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330243:330243:256:4294967295] 192.168.202.110@tcp LPNI seq info [330200:330200:8:4294967295]
00000400:00000200:0.0:1788574331.285954:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574331.285969:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574331.285978:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574331.285987:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574331.286009:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574331.286017:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574331.286029:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574331.286038:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574331.286049:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574331.286054:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574331.286062:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574331.286068:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574331.286075:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574331.286079:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574331.286084:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574331.286092:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286113:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574331.286121:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574356 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286141:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574331.286146:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574331.286154:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574331.286161:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574331.286166:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305935801
00000100:00000010:0.0:1788574331.286174:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574331.286182:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574331.286189:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305935801
00000100:00000010:0.0:1788574331.286195:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7cc0.
00000100:00000001:0.0:1788574331.286201:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574331.286205:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574331.286208:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574331.286221:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574331.286229:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574331/real 1788574331]  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574331.286250:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574331.286253:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574331.286258:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574331.286264:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574331.286267:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574331.286271:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574331.286277:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574331.286286:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286292:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574331.286294:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286296:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574331.286297:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574331.286300:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286305:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574331.286308:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286313:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574331.286315:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286316:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574331.286317:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286318:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574331.286320:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574331.286322:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574331.286324:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574331.286326:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574331.286328:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574331.286334:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393126400:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574331.286337:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286339:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574331.286341:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393126400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286345:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574331.286347:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574331.286349:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574331.286351:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574331.286352:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574331.286354:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574331.286356:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574331.286357:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788574331.286360:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574331.286362:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574331.286363:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:0.0:1788574331.286366:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574331.286367:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286368:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286374:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574331.286421:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574331.286425:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574331.286427:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574331.286428:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574331.286433:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286440:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574331.286442:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574331.286444:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574331.286446:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574331.286448:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574331.286451:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574331.286455:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574331.286457:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574331.286459:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574331.286461:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574331.286464:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574331.286466:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574331.286470:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574331.286474:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574331.286478:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574331.286482:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393126528, portal 10
00000100:00000001:0.0:1788574331.286484:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574331.286486:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:0.0:1788574331.286493:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286500:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574331.286503:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574331.286505:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574331.286508:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393126528, offset 0
00000400:00000200:0.0:1788574331.286512:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574331.286517:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574331.286521:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574331.286524:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574331.286527:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574331.286530:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574331.286533:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574331.286536:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330244:330244:256:4294967295] 192.168.202.110@tcp LPNI seq info [330201:330201:8:4294967295]
00000400:00000200:0.0:1788574331.286540:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574331.286547:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574331.286553:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574331.286557:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574331.286561:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574331.286564:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574331.286568:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574331.286570:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574331.286573:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574331.286574:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574331.286577:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574331.286579:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574331.286582:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574331.286583:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574331.286585:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574331.286588:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286593:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574331.286595:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574356 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286605:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574331.286606:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574331.286608:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574331.286611:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574331.286614:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574331.286615:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305934887
00000100:00000001:0.0:1788574331.286617:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305935801
00000100:00000001:0.0:1788574331.286619:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305934887
00000100:00000001:0.0:1788574331.286621:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305935801
00000100:00000001:0.0:1788574331.286622:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574331.286623:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574331.286625:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574331.286627:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574331/real 1788574331]  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574331.286633:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574331.286634:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574331.286635:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574331.286637:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574331.286638:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574331.286639:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574331.286641:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574331.286647:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286651:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574331.286652:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286653:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574331.286654:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574331.286656:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286661:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574331.286664:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286668:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574331.286669:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286670:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574331.286671:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286672:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574331.286674:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574331.286675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574331.286677:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574331.286679:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574331.286681:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574331.286686:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574331.286689:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574331.286691:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286695:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574331.286696:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574331.286698:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574331.286699:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574331.286700:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574331.286702:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574331.286703:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574331.286705:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574331.286706:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574331.286708:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574331.286710:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574331.286711:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574331.286714:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574331.286716:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574331.286718:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574331.286720:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393126656, portal 4
00000100:00000001:0.0:1788574331.286721:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574331.286722:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:0.0:1788574331.286726:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286730:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574331.286732:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574331.286733:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574331.286735:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393126656, offset 0
00000400:00000200:0.0:1788574331.286737:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574331.286739:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574331.286742:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574331.286744:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574331.286745:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574331.286747:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574331.286748:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574331.286751:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330245:330245:256:4294967295] 192.168.202.110@tcp LPNI seq info [330202:330202:8:4294967295]
00000400:00000200:0.0:1788574331.286753:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574331.286757:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574331.286761:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574331.286762:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574331.286765:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574331.286767:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574331.286770:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574331.286771:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574331.286774:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574331.286776:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574331.286777:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574331.286780:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574331.286782:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574331.286783:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574331.286784:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574331.286786:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286791:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574331.286793:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574356 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286797:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574331.286798:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574331.286800:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574331.286802:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574331.286804:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574331.286805:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305935033
00000100:00000001:0.0:1788574331.286807:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305935801
00000100:00000001:0.0:1788574331.286808:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305935033
00000100:00000001:0.0:1788574331.286810:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305935801
00000100:00000001:0.0:1788574331.286811:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574331.286812:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574331.286813:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574331.286816:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574331/real 1788574331]  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574331.286820:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574331.286821:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574331.286822:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574331.286824:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574331.286825:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574331.286826:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574331.286828:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574331.286834:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286838:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574331.286839:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286840:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574331.286841:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574331.286843:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286847:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574331.286849:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286853:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574331.286854:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286855:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574331.286856:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286857:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574331.286858:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574331.286860:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574331.286862:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574331.286864:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574331.286866:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574331.286871:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574331.286873:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286874:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574331.286877:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393126528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286881:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574331.286882:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574331.286884:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574331.286885:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574331.286886:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574331.286887:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574331.286888:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574331.286889:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788574331.286891:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574331.286892:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574331.286893:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:0.0:1788574331.286895:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574331.286896:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286897:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574331.286899:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393126656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574356 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574331.286904:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574331.286904:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574331.286905:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574331.286907:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574331.286907:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574331.286908:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574331.286909:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574331.286910:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788574331.286911:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574331.286912:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574331.286912:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:0.0:1788574331.286914:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574331.286915:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286916:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574331.286919:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574331.286919:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574331.286921:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574331.286922:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574331.669535:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574331.669540:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574331.669544:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574331.797464:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574331.797472:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574331.797474:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574331.797480:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574331.797484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574331.797488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574331.797491:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574331.797494:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574331.797498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574332.245490:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305935801, at 4305935808
00000100:00000040:1.0:1788574332.245490:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305935801, at 4305935808
00000100:00000010:1.0:1788574332.245505:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7cc0.
00000100:00000010:0.0:1788574332.245507:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:1.0:1788574332.821476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574332.821484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574332.821487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574332.821494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574332.821498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574332.821503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574332.821506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574332.821510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574332.821513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574333.717609:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574333.717612:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574333.717615:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574333.845514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574333.845522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574333.845525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574333.845530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574333.845533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574333.845537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574333.845539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574333.845542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574333.845545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574334.741509:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574334.741514:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574334.741518:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574334.869471:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574334.869478:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574334.869480:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574334.869484:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574334.869487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574334.869490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574334.869492:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574334.869494:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574334.869496:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574335.893509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574335.893516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574335.893518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574335.893524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574335.893528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574335.893532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574335.893535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574335.893537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574335.893541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574336.405807:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574336.405813:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574336.405819:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574336.405831:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.405851:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574336.405856:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574336.405863:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574336.405866:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574336.405869:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574336.405876:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393126784:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574336.405886:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574336.405893:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574336.405896:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574336.405904:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574336.405911:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574336.405915:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574336.405936:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574336.405944:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574336.405956:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574336.405966:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393126784, portal 25
00000100:00000001:0.0:1788574336.405972:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574336.405976:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:0.0:1788574336.405989:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406005:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574336.406021:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574336.406027:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574336.406033:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393126784, offset 0
00000400:00000200:0.0:1788574336.406044:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574336.406056:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574336.406067:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574336.406077:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574336.406086:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574336.406095:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574336.406102:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574336.406110:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330246:330246:256:4294967295] 192.168.202.110@tcp LPNI seq info [330203:330203:8:4294967295]
00000400:00000200:0.0:1788574336.406122:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574336.406139:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574336.406151:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574336.406161:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574336.406173:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574336.406179:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788574336.406191:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574336.406198:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1788574336.406206:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574336.406211:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574336.406219:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574336.406227:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574336.406235:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574336.406241:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574336.406248:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574336.406256:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406293:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574336.406301:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574361 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406319:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574336.406323:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574336.406331:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574336.406340:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574336.406348:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305940921
00000100:00000010:0.0:1788574336.406357:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574336.406364:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574336.406372:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305940921
00000100:00000010:0.0:1788574336.406376:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7c00.
00000100:00000001:0.0:1788574336.406417:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574336.406422:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574336.406441:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574336.406444:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574336.406449:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574336/real 1788574336]  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574336.406460:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574336.406462:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574336.406465:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574336.406468:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574336.406470:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574336.406473:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574336.406477:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574336.406486:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406502:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574336.406504:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.406507:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574336.406509:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574336.406514:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406522:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574336.406527:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406542:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574336.406545:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.406546:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574336.406548:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.406550:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574336.406553:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574336.406556:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574336.406566:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574336.406569:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574336.406574:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574336.406583:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393126784:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574336.406589:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.406591:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574336.406594:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393126784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406601:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574336.406603:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574336.406606:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574336.406610:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574336.406612:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574336.406614:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574336.406616:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574336.406619:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:0.0:1788574336.406623:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574336.406625:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574336.406627:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:0.0:1788574336.406630:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574336.406632:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.406634:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.406642:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574336.406644:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574336.406647:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574336.406649:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574336.406650:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574336.406655:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406662:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574336.406664:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574336.406666:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574336.406668:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574336.406671:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574336.406674:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126912:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574336.406678:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574336.406680:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574336.406681:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574336.406683:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574336.406685:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574336.406687:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574336.406690:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574336.406692:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574336.406695:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574336.406697:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393126912, portal 10
00000100:00000001:0.0:1788574336.406699:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574336.406700:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000100:00000040:0.0:1788574336.406705:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406710:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574336.406711:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574336.406713:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574336.406714:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393126912, offset 0
00000400:00000200:0.0:1788574336.406717:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574336.406720:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574336.406723:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574336.406726:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574336.406727:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574336.406729:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574336.406731:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574336.406733:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330247:330247:256:4294967295] 192.168.202.110@tcp LPNI seq info [330204:330204:8:4294967295]
00000400:00000200:0.0:1788574336.406736:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574336.406740:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574336.406744:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574336.406746:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574336.406749:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574336.406751:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788574336.406754:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574336.406756:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1788574336.406758:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574336.406760:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574336.406762:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574336.406764:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574336.406767:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574336.406768:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574336.406770:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574336.406772:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406778:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574336.406780:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574361 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406784:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574336.406786:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574336.406788:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574336.406790:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574336.406792:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574336.406794:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305940007
00000100:00000001:0.0:1788574336.406796:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305940921
00000100:00000001:0.0:1788574336.406797:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305940007
00000100:00000001:0.0:1788574336.406799:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305940921
00000100:00000001:0.0:1788574336.406800:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574336.406802:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574336.406803:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574336.406805:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574336/real 1788574336]  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574336.406810:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574336.406811:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574336.406812:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574336.406813:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574336.406815:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574336.406816:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574336.406818:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574336.406823:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406827:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574336.406828:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.406829:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574336.406830:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574336.406832:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406836:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574336.406838:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406843:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574336.406844:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.406846:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574336.406847:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.406848:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574336.406851:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574336.406853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574336.406856:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574336.406859:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574336.406863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574336.406872:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393126912:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574336.406876:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574336.406880:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406887:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574336.406888:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574336.406891:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574336.406893:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574336.406895:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574336.406898:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393127040:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574336.406902:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574336.406904:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574336.406906:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574336.406907:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574336.406911:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574336.406913:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574336.406917:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574336.406920:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574336.406923:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574336.406927:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393127040, portal 4
00000100:00000001:0.0:1788574336.406929:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574336.406931:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:0.0:1788574336.406937:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.406943:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574336.406944:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574336.406946:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574336.406948:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393127040, offset 0
00000400:00000200:0.0:1788574336.406950:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574336.406953:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574336.406956:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574336.406958:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574336.406960:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574336.406962:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574336.406964:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574336.406966:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330248:330248:256:4294967295] 192.168.202.110@tcp LPNI seq info [330205:330205:8:4294967295]
00000400:00000200:0.0:1788574336.406969:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574336.406973:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574336.406976:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574336.406978:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574336.406981:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574336.406983:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788574336.406986:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574336.406988:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1788574336.406990:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574336.406992:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574336.406994:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574336.406996:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574336.406998:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574336.406999:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574336.407001:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574336.407004:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.407009:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574336.407011:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574361 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.407016:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574336.407017:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574336.407019:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574336.407021:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574336.407023:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574336.407024:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305940153
00000100:00000001:0.0:1788574336.407026:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305940921
00000100:00000001:0.0:1788574336.407028:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305940153
00000100:00000001:0.0:1788574336.407029:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305940921
00000100:00000001:0.0:1788574336.407031:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574336.407032:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574336.407033:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574336.407035:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574336/real 1788574336]  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574336.407040:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574336.407041:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574336.407042:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574336.407044:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574336.407045:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574336.407046:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574336.407048:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574336.407054:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.407057:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574336.407058:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.407059:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574336.407060:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574336.407063:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.407067:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574336.407070:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.407075:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574336.407077:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.407079:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574336.407080:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.407082:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574336.407083:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574336.407086:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574336.407089:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574336.407092:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574336.407096:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574336.407105:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393127040:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574336.407109:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.407111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574336.407114:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393126912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.407122:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574336.407123:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574336.407125:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574336.407128:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574336.407130:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574336.407131:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574336.407134:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574336.407136:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:0.0:1788574336.407138:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574336.407139:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574336.407140:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:0.0:1788574336.407142:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574336.407143:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.407144:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574336.407146:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393127040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574361 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574336.407151:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574336.407152:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574336.407153:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574336.407154:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574336.407155:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574336.407156:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574336.407157:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574336.407158:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788574336.407159:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574336.407160:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574336.407161:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788574336.407163:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574336.407164:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.407165:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574336.407168:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574336.407168:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574336.407170:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574336.407171:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574336.789593:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574336.789598:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574336.789602:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574336.917553:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574336.917560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574336.917562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574336.917567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574336.917571:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574336.917575:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574336.917578:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574336.917581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574336.917584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574337.365506:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305940921, at 4305940928
00000100:00000040:1.0:1788574337.365506:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305940921, at 4305940928
00000100:00000010:1.0:1788574337.365525:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7c00.
00000100:00000010:0.0:1788574337.365530:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:1.0:1788574337.813631:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574337.813634:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574337.813637:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574337.941477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574337.941484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574337.941486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574337.941491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574337.941494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574337.941497:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574337.941499:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574337.941500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574337.941503:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574338.965522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574338.965534:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574338.965536:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574338.965541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574338.965544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574338.965547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574338.965549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574338.965550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574338.965553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574339.861643:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574339.861646:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574339.861649:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574339.989502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574339.989509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574339.989512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574339.989517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574339.989528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574339.989532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574339.989539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574339.989541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574339.989545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574340.885581:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574340.885586:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574340.885589:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574341.013522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574341.013530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574341.013533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574341.013539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574341.013544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574341.013548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574341.013551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574341.013554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574341.013557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574341.525501:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574341.525510:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574341.525512:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574341.525515:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574341.525519:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574341.525525:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11272
00000400:00000200:1.0:1788574341.525534:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574341.525540:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574341.525544:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574341.525547:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525550:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574341.525553:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525556:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574341.525560:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574341.525570:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574341.525574:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574341.525577:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574341.525579:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574341.525581:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574341.525584:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525585:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574341.525587:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525588:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574341.525590:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574341.525593:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525594:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574341.525597:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574341.525600:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574341.525603:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574341.525604:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574341.525606:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574341.525614:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788574341.525617:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574341.525620:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574341.525621:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574341.525623:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574341.525625:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574341.525628:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574341.525634:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788574341.525640:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574341.525647:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574341.525656:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574341.525659:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574341.525663:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574341.525680:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574341.525684:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574341.525687:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574341.525689:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574341.525690:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574341.525694:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574341.525697:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11272
00000400:00000200:1.0:1788574341.525700:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574341.525704:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574341.525706:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574341.525709:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525711:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574341.525714:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525716:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788574341.525717:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1788574341.525718:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574341.525719:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574341.525721:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788574341.525722:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:1.0:1788574341.525723:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:0.0:1788574341.525724:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1788574341.525725:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574341.525727:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574341.525731:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:0.0:1788574341.525732:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574341.525733:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574341.525735:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525737:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574341.525740:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788574341.525741:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:1.0:1788574341.525742:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574341.525744:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574341.525745:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574341.525747:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574341.525747:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574341.525750:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574341.525752:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574341.525753:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574341.525755:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574341.525756:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393127168:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574341.525757:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574341.525760:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:1.0:1788574341.525761:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574341.525764:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574341.525765:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1788574341.525766:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574341.525767:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574341.525769:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:1.0:1788574341.525769:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788574341.525772:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000010:0.0:1788574341.525774:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:1.0:1788574341.525774:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574341.525776:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574341.525776:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574341.525778:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574341.525779:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574341.525781:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574341.525782:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:1.0:1788574341.525784:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000400:00000010:0.0:1788574341.525786:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000100:00000001:1.0:1788574341.525788:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574341.525793:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00100000:1.0:1788574341.525793:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1788574341.525799:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393127168, portal 25
00000100:00000001:1.0:1788574341.525800:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574341.525801:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574341.525802:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.525803:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345471232 : -130483364080384 : ffff8953796f5500)
00000100:00000040:1.0:1788574341.525806:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574341.525809:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574341.525816:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.525818:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:1.0:1788574341.525819:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574341.525821:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788574341.525822:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:0.0:1788574341.525824:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000001:1.0:1788574341.525824:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:0.0:1788574341.525827:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393127168, offset 0
00000400:00000200:0.0:1788574341.525833:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788574341.525835:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574341.525837:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574341.525839:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11272
00000400:00000200:0.0:1788574341.525841:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574341.525848:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574341.525853:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574341.525858:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574341.525862:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574341.525864:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574341.525868:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330249:330249:256:4294967295] 192.168.202.110@tcp LPNI seq info [330206:330206:8:4294967295]
00000400:00000200:0.0:1788574341.525872:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574341.525882:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:1.0:1788574341.525887:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:0.0:1788574341.525888:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574341.525890:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574341.525892:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000010:0.0:1788574341.525894:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000100:00000040:1.0:1788574341.525895:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525901:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574341.525903:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:0.0:1788574341.525905:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574341.525905:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:0.0:1788574341.525908:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:1.0:1788574341.525908:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574341.525909:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574341.525911:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574341.525913:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574341.525915:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:0.1:1788574341.525915:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788574341.525917:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:0.1:1788574341.525919:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000040:1.0:1788574341.525920:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574341.525922:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000100:0.0:1788574341.525924:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788574341.525925:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:0.0:1788574341.525927:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000100:00000001:1.0:1788574341.525927:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574341.525929:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574341.525932:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1788574341.525933:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1788574341.525935:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574341.525937:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1788574341.525938:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:1.0:1788574341.525939:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574341.525941:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574341.525943:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1788574341.525944:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000008:00100000:1.0:1788574341.525945:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000010:0.0:1788574341.525946:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000008:00000001:1.0:1788574341.525948:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.525950:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:1.0:1788574341.525950:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574341.525953:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000100:00000200:0.0:1788574341.525955:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788574341.525956:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574341.525958:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574341.525960:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574341.525962:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574341.525963:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1788574341.525965:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:1.0:1788574341.525965:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574341.525968:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574366 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788574341.525969:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788574341.525972:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.525976:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574341.525976:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574341.525978:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574341.525983:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574341.525984:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574341.525985:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574341.525988:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788574341.525989:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.525991:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305946041
00000100:00000010:0.0:1788574341.525996:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:1.0:1788574341.525997:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574341.526000:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1788574341.526003:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:1.0:1788574341.526003:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574341.526008:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11282)
00000100:00000001:0.0:1788574341.526010:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305946041
00000100:00000010:0.0:1788574341.526012:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788574341.526016:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574341.526019:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574341.526022:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.526024:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574341.526029:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574341/real 1788574341]  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574341.526040:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574341.526041:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574341.526044:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574341.526047:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574341.526049:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574341.526051:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574341.526055:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574341.526065:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526070:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574341.526071:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526073:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574341.526075:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574341.526087:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526095:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574341.526100:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526107:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574341.526110:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526112:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574341.526114:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526116:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574341.526119:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574341.526122:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574341.526126:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574341.526130:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574341.526135:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574341.526142:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393127168:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574341.526146:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526148:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574341.526150:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5500 x1875444393127168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526154:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574341.526157:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574341.526159:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574341.526162:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574341.526163:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574341.526164:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574341.526166:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574341.526168:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:0.0:1788574341.526171:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574341.526172:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574341.526173:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5500.
00000100:00000001:0.0:1788574341.526175:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574341.526176:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526178:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526184:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574341.526186:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.526188:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574341.526189:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574341.526190:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574341.526192:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526201:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574341.526202:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574341.526204:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574341.526205:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574341.526207:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574341.526208:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393127296:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574341.526210:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574341.526212:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574341.526213:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574341.526214:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574341.526216:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574341.526218:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574341.526221:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574341.526223:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574341.526225:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574341.526228:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393127296, portal 10
00000100:00000001:0.0:1788574341.526229:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574341.526230:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:0.0:1788574341.526234:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526239:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574341.526240:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574341.526242:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574341.526243:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393127296, offset 0
00000400:00000200:0.0:1788574341.526246:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574341.526249:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574341.526252:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574341.526259:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574341.526261:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574341.526263:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574341.526265:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574341.526267:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330250:330250:256:4294967295] 192.168.202.110@tcp LPNI seq info [330207:330207:8:4294967295]
00000400:00000200:0.0:1788574341.526271:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574341.526275:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574341.526279:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574341.526281:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574341.526284:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574341.526286:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788574341.526289:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574341.526291:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1788574341.526293:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574341.526295:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574341.526297:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574341.526299:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574341.526301:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574341.526303:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574341.526304:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574341.526307:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526312:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574341.526314:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574366 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526319:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.526320:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574341.526322:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574341.526324:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574341.526326:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574341.526328:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305945126
00000100:00000001:0.0:1788574341.526330:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305946041
00000100:00000001:0.0:1788574341.526331:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305945126
00000100:00000001:0.0:1788574341.526333:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305946041
00000100:00000001:0.0:1788574341.526334:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574341.526336:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.526337:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574341.526340:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574341/real 1788574341]  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574341.526345:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574341.526346:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574341.526347:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574341.526348:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574341.526350:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574341.526351:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574341.526353:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574341.526359:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526363:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574341.526363:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526364:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574341.526365:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574341.526368:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526372:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574341.526374:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526378:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574341.526379:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526380:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574341.526382:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526383:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574341.526384:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574341.526386:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574341.526389:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574341.526421:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574341.526425:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574341.526431:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393127296:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574341.526433:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574341.526435:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526440:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574341.526441:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574341.526442:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574341.526444:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574341.526445:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574341.526447:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393127424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574341.526449:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574341.526450:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574341.526451:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574341.526452:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574341.526455:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574341.526456:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574341.526459:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574341.526460:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574341.526462:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574341.526465:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393127424, portal 4
00000100:00000001:0.0:1788574341.526466:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574341.526467:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:0.0:1788574341.526471:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526490:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574341.526492:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574341.526494:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574341.526496:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393127424, offset 0
00000400:00000200:0.0:1788574341.526499:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574341.526502:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574341.526504:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574341.526506:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574341.526508:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574341.526510:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574341.526511:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574341.526514:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330251:330251:256:4294967295] 192.168.202.110@tcp LPNI seq info [330208:330208:8:4294967295]
00000400:00000200:0.0:1788574341.526516:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574341.526520:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574341.526523:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574341.526525:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574341.526528:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574341.526530:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1788574341.526533:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574341.526535:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1788574341.526537:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574341.526539:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574341.526541:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574341.526543:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574341.526546:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574341.526547:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574341.526548:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574341.526551:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526556:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574341.526559:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574366 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526563:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.526564:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574341.526567:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574341.526569:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574341.526573:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574341.526575:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305945273
00000100:00000001:0.0:1788574341.526577:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305946041
00000100:00000001:0.0:1788574341.526579:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305945273
00000100:00000001:0.0:1788574341.526582:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305946041
00000100:00000001:0.0:1788574341.526584:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574341.526586:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.526588:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574341.526592:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574341/real 1788574341]  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574341.526601:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574341.526602:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574341.526604:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574341.526607:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574341.526609:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574341.526611:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574341.526614:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574341.526623:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526628:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574341.526629:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526631:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574341.526632:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574341.526634:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526639:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574341.526641:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526646:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574341.526648:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526649:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574341.526650:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526651:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574341.526652:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574341.526654:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574341.526656:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574341.526657:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574341.526659:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574341.526664:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393127424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574341.526667:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526668:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574341.526670:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393127296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526674:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574341.526675:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574341.526677:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574341.526678:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574341.526679:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574341.526680:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574341.526681:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574341.526683:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:0.0:1788574341.526684:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574341.526685:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574341.526686:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:0.0:1788574341.526688:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574341.526689:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526690:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574341.526692:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393127424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574366 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574341.526696:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574341.526697:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574341.526698:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574341.526699:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574341.526700:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574341.526701:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574341.526702:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574341.526703:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:0.0:1788574341.526704:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574341.526704:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574341.526705:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:0.0:1788574341.526707:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574341.526707:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526708:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574341.526711:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574341.526712:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574341.526713:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574341.526714:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574342.037502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574342.037508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574342.037510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574342.037515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574342.037518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574342.037521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574342.037523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574342.037525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574342.037527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574342.485516:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305946041, at 4305946048
00000100:00000040:0.0:1788574342.485518:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305946041, at 4305946048
00000100:00000010:0.0:1788574342.485529:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574342.485534:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000020:00000001:0.0:1788574342.933588:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574342.933591:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574342.933594:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574343.061517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574343.061523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574343.061524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574343.061528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574343.061531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574343.061534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574343.061536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574343.061538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574343.061540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574343.957600:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574343.957605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574343.957609:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574344.085483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574344.085490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574344.085491:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574344.085496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574344.085510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574344.085515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574344.085518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574344.085520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574344.085523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574345.109503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574345.109539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574345.109541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574345.109546:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574345.109549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574345.109552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574345.109554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574345.109557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574345.109558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574346.005593:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574346.005596:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574346.005600:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574346.133486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574346.133494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574346.133496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574346.133502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574346.133506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574346.133509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574346.133512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574346.133515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574346.133532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574346.645512:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574346.645539:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574346.645546:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574346.645550:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574346.645554:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574346.645562:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11277
00000400:00000200:0.0:1788574346.645571:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574346.645577:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574346.645581:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574346.645584:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645587:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574346.645590:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645592:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574346.645597:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574346.645599:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574346.645601:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574346.645604:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574346.645607:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574346.645610:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574346.645613:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645615:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574346.645617:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645619:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574346.645621:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574346.645625:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645627:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574346.645630:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574346.645634:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574346.645637:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574346.645639:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574346.645641:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574346.645651:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788574346.645655:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574346.645659:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574346.645660:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574346.645663:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574346.645666:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574346.645670:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574346.645678:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788574346.645684:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574346.645694:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393127552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574346.645705:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574346.645709:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574346.645714:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393127552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574346.645738:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574346.645742:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574346.645746:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574346.645747:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574346.645749:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574346.645752:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574346.645755:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11277
00000400:00000200:0.0:1788574346.645759:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574346.645763:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574346.645765:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574346.645768:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645771:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574346.645773:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645777:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574346.645780:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574346.645784:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574346.645786:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574346.645790:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574346.645794:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574346.645798:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574346.645804:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645810:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574346.645814:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645819:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574346.645825:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574346.645831:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574346.645836:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574346.645840:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574346.645849:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574346.645853:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574346.645858:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574346.645863:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574346.645871:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574346.645874:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574346.645881:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788574346.645886:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574346.645893:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574346.645897:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574346.645899:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574346.645901:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574346.645904:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574346.645912:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574346.645921:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574346.645928:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393127680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574346.645936:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574346.645939:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574346.645944:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393127680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574346.645955:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574346.645959:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574346.645966:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574346.645967:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574346.645969:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574346.645973:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574346.645978:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11277
00000400:00000200:0.0:1788574346.645986:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574346.645991:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574346.645996:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574346.646001:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.646003:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574346.646008:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.646013:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574346.646016:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574346.646017:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574346.646021:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574346.646024:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574346.646026:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574346.646030:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574346.646033:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.646034:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574346.646037:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574346.646040:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574346.646043:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574346.646049:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574346.646051:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574346.646052:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574346.646054:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574346.646057:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574346.646062:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574346.646065:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574346.646067:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574346.646071:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574346.646076:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788574346.646079:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574346.646082:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574346.646085:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574346.646087:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574346.646089:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574346.646091:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574346.646097:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574346.646102:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574346.646110:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393127808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574346.646122:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574346.646123:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574346.646131:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393127808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574346.646143:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574346.646145:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574346.646148:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574346.646154:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11287)
00000800:00000200:2.1:1788574346.646199:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574346.646203:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788574346.646723:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574346.646726:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788574346.647040:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574346.647042:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:1.0:1788574347.029573:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574347.029576:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574347.029578:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574347.157465:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574347.157472:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574347.157474:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574347.157479:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574347.157482:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574347.157485:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574347.157487:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574347.157489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574347.157491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574347.605489:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305951161, at 4305951168
00000100:00000040:1.0:1788574347.605489:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305951161, at 4305951168
00000100:00000010:1.0:1788574347.605503:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00000100:00000010:0.0:1788574347.605506:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000100:00000001:1.0:1788574348.181505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574348.181513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574348.181515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574348.181521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574348.181525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574348.181530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574348.181532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574348.181565:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574348.181569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574349.077612:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574349.077617:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574349.077621:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574349.205490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574349.205496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574349.205497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574349.205502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574349.205505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574349.205508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574349.205510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574349.205512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574349.205514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574350.101602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574350.101605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574350.101608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574350.229470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574350.229476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574350.229478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574350.229483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574350.229486:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574350.229489:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574350.229491:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574350.229493:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574350.229495:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574351.253460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574351.253467:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574351.253468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574351.253472:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574351.253475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574351.253478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574351.253480:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574351.253482:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574351.253484:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574351.766164:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574351.766171:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574351.766593:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574351.766595:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574351.766798:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574351.766799:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1788574352.149546:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574352.149550:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574352.149554:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574352.277490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574352.277495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574352.277497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574352.277502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574352.277505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574352.277508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574352.277510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574352.277512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574352.277514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574352.725465:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305956281, at 4305956288
00000100:00000040:0.0:1788574352.725468:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305956281, at 4305956288
00000100:00000010:0.0:1788574352.725479:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000100:00000010:1.0:1788574352.725485:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000020:00000001:1.0:1788574353.173629:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788574353.173634:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788574353.173637:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574353.301518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574353.301526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574353.301529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574353.301535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574353.301539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574353.301543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574353.301546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574353.301548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574353.301551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574354.325466:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574354.325473:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574354.325474:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574354.325480:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574354.325483:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574354.325486:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574354.325489:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574354.325491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574354.325493:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574355.221655:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574355.221659:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574355.221662:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574355.349558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574355.349566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574355.349568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574355.349574:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574355.349578:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574355.349582:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574355.349584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574355.349586:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574355.349588:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574356.373497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574356.373505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574356.373507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574356.373521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574356.373525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574356.373528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574356.373531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574356.373532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574356.373535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574356.885776:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574356.885783:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574356.885787:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574356.885799:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.885828:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574356.885832:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574356.885838:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574356.885843:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574356.885849:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574356.885855:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393128320:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574356.885861:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574356.885867:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574356.885872:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574356.885878:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574356.885886:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574356.885892:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574356.885902:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574356.885909:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574356.885919:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574356.885929:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393128320, portal 25
00000100:00000001:0.0:1788574356.885936:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574356.885939:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:0.0:1788574356.885951:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.885983:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574356.885991:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574356.885998:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574356.886003:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393128320, offset 0
00000400:00000200:0.0:1788574356.886018:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574356.886031:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574356.886043:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574356.886052:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574356.886060:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574356.886068:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574356.886076:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574356.886086:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330258:330258:256:4294967295] 192.168.202.110@tcp LPNI seq info [330215:330215:8:4294967295]
00000400:00000200:0.0:1788574356.886098:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574356.886116:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574356.886128:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574356.886137:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574356.886150:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574356.886159:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574356.886170:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574356.886177:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574356.886186:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574356.886191:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574356.886200:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574356.886211:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574356.886218:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574356.886224:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574356.886232:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574356.886241:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886260:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574356.886265:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574381 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886274:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574356.886276:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574356.886281:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574356.886286:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574356.886289:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305961401
00000100:00000010:0.0:1788574356.886295:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574356.886301:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574356.886308:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305961401
00000100:00000010:0.0:1788574356.886311:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d78a0.
00000100:00000001:0.0:1788574356.886314:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574356.886317:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574356.886320:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574356.886323:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574356.886328:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574356/real 1788574356]  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574356.886338:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574356.886340:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574356.886343:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574356.886346:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574356.886348:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574356.886351:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574356.886355:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574356.886366:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886374:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574356.886376:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.886378:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574356.886381:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574356.886385:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886425:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574356.886431:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886449:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574356.886456:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.886458:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574356.886460:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.886462:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574356.886465:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574356.886468:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574356.886472:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574356.886474:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574356.886479:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574356.886492:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393128320:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574356.886497:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.886499:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574356.886503:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393128320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886509:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574356.886511:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574356.886514:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574356.886517:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574356.886518:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574356.886520:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574356.886522:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574356.886524:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788574356.886527:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574356.886529:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574356.886531:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:0.0:1788574356.886534:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574356.886535:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.886537:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.886544:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574356.886545:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574356.886547:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574356.886549:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574356.886550:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574356.886554:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886560:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574356.886562:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574356.886564:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574356.886566:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574356.886568:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574356.886571:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393128448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574356.886574:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574356.886576:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574356.886577:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574356.886579:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574356.886582:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574356.886584:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574356.886588:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574356.886592:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574356.886595:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574356.886598:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393128448, portal 10
00000100:00000001:0.0:1788574356.886600:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574356.886601:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:0.0:1788574356.886608:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886615:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574356.886633:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574356.886637:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574356.886640:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393128448, offset 0
00000400:00000200:0.0:1788574356.886645:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574356.886649:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574356.886654:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574356.886657:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574356.886660:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574356.886662:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574356.886665:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574356.886668:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330259:330259:256:4294967295] 192.168.202.110@tcp LPNI seq info [330216:330216:8:4294967295]
00000400:00000200:0.0:1788574356.886672:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574356.886678:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574356.886684:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574356.886688:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574356.886692:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574356.886695:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574356.886700:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574356.886703:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574356.886707:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574356.886709:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574356.886713:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574356.886716:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574356.886720:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574356.886722:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574356.886725:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574356.886728:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886737:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574356.886740:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574381 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886747:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574356.886748:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574356.886751:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574356.886754:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574356.886758:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574356.886760:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305960487
00000100:00000001:0.0:1788574356.886763:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305961401
00000100:00000001:0.0:1788574356.886765:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305960487
00000100:00000001:0.0:1788574356.886767:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305961401
00000100:00000001:0.0:1788574356.886769:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574356.886771:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574356.886774:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574356.886777:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574356/real 1788574356]  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574356.886786:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574356.886787:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574356.886789:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574356.886792:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574356.886794:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574356.886796:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574356.886800:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574356.886808:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886815:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574356.886817:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.886819:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574356.886820:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574356.886824:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886832:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574356.886836:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886844:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574356.886847:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.886849:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574356.886850:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.886852:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574356.886855:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574356.886858:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574356.886862:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574356.886864:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574356.886869:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574356.886878:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393128448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574356.886883:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574356.886886:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886893:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574356.886895:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574356.886897:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574356.886899:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574356.886902:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574356.886905:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393128576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574356.886908:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574356.886910:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574356.886912:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574356.886914:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574356.886917:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574356.886920:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574356.886923:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574356.886926:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574356.886929:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574356.886933:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393128576, portal 4
00000100:00000001:0.0:1788574356.886935:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574356.886936:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:0.0:1788574356.886942:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.886950:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574356.886953:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574356.886955:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574356.886957:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393128576, offset 0
00000400:00000200:0.0:1788574356.886961:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574356.886965:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574356.886968:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574356.886970:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574356.886972:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574356.886974:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574356.886976:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574356.886978:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330260:330260:256:4294967295] 192.168.202.110@tcp LPNI seq info [330217:330217:8:4294967295]
00000400:00000200:0.0:1788574356.886981:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574356.886986:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574356.886989:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574356.886991:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574356.886994:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574356.886996:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574356.886999:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574356.887001:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574356.887003:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574356.887005:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574356.887007:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574356.887009:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574356.887012:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574356.887013:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574356.887015:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574356.887017:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.887023:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574356.887025:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574381 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.887029:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574356.887030:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574356.887033:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574356.887035:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574356.887037:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574356.887039:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305960633
00000100:00000001:0.0:1788574356.887040:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305961401
00000100:00000001:0.0:1788574356.887042:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305960633
00000100:00000001:0.0:1788574356.887043:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305961401
00000100:00000001:0.0:1788574356.887045:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574356.887047:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574356.887048:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574356.887050:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574356/real 1788574356]  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574356.887055:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574356.887056:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574356.887057:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574356.887058:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574356.887059:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574356.887060:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574356.887062:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574356.887067:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.887071:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574356.887073:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.887074:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574356.887075:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574356.887077:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.887081:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574356.887083:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.887088:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574356.887090:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.887091:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574356.887092:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.887093:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574356.887094:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574356.887096:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574356.887098:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574356.887100:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574356.887105:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574356.887113:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393128576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574356.887117:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.887119:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574356.887122:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393128448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.887129:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574356.887131:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574356.887133:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574356.887136:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574356.887137:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574356.887139:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574356.887141:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574356.887143:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788574356.887146:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574356.887147:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574356.887149:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:0.0:1788574356.887152:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574356.887153:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.887155:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574356.887159:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393128576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574381 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574356.887165:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574356.887166:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574356.887168:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574356.887170:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574356.887171:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574356.887173:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574356.887174:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574356.887176:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788574356.887178:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574356.887179:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574356.887180:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:0.0:1788574356.887183:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574356.887184:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.887186:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574356.887190:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574356.887192:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574356.887194:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574356.887195:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574357.397490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574357.397497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574357.397499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574357.397505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574357.397509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574357.397512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574357.397515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574357.397518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574357.397521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574357.845510:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305961401, at 4305961408
00000100:00000040:0.0:1788574357.845511:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305961401, at 4305961408
00000100:00000010:1.0:1788574357.845527:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d78a0.
00000100:00000010:0.0:1788574357.845535:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788574358.293663:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574358.293668:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574358.293673:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574358.421546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574358.421561:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574358.421564:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574358.421572:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574358.421577:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574358.421583:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574358.421587:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574358.421591:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574358.421595:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574361.365560:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574361.365563:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574361.365566:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788574361.493503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574361.493509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574361.493511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574361.493516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574361.493520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574361.493523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574361.493526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574361.493529:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574361.493531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.005720:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574362.005725:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574362.005727:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574362.005734:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.005742:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574362.005745:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574362.005748:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574362.005751:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574362.005753:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574362.005757:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393128704:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574362.005761:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574362.005765:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574362.005768:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574362.005770:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574362.005774:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574362.005776:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574362.005782:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574362.005786:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574362.005792:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574362.005798:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393128704, portal 25
00000100:00000001:0.0:1788574362.005801:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574362.005803:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:0.0:1788574362.005809:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.005817:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574362.005820:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574362.005823:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574362.005826:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393128704, offset 0
00000400:00000200:0.0:1788574362.005832:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574362.005839:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574362.005846:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574362.005852:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574362.005857:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574362.005861:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574362.005864:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574362.005868:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330261:330261:256:4294967295] 192.168.202.110@tcp LPNI seq info [330218:330218:8:4294967295]
00000400:00000200:0.0:1788574362.005874:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574362.005883:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574362.005889:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574362.005895:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574362.005901:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574362.005905:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788574362.005912:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574362.005915:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788574362.005920:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574362.005923:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574362.005929:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574362.005934:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574362.005939:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574362.005942:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574362.005946:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574362.005950:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.005960:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574362.005964:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.005970:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.005972:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574362.005977:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574362.005979:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574362.005982:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305966521
00000100:00000010:0.0:1788574362.005986:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574362.005991:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574362.005996:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305966521
00000100:00000010:0.0:1788574362.005998:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7a80.
00000100:00000001:0.0:1788574362.006000:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574362.006002:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574362.006004:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.006006:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574362.006009:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574362/real 1788574362]  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574362.006016:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574362.006018:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574362.006019:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574362.006021:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574362.006023:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574362.006024:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574362.006027:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574362.006032:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006036:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574362.006037:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006048:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574362.006049:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574362.006052:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006056:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574362.006060:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006065:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574362.006067:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006068:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574362.006069:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006073:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574362.006075:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574362.006077:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574362.006080:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574362.006081:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574362.006084:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574362.006089:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393128704:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574362.006093:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006094:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574362.006096:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393128704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006100:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574362.006102:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574362.006104:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574362.006106:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574362.006108:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574362.006109:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574362.006110:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574362.006112:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788574362.006116:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574362.006117:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574362.006118:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:0.0:1788574362.006121:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574362.006122:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006123:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006129:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574362.006130:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.006131:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574362.006132:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574362.006133:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574362.006136:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006141:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574362.006142:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574362.006143:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574362.006145:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574362.006146:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574362.006148:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393128832:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574362.006150:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574362.006151:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574362.006152:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574362.006153:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574362.006155:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574362.006156:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574362.006159:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574362.006161:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574362.006163:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574362.006165:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393128832, portal 10
00000100:00000001:0.0:1788574362.006167:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574362.006168:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:0.0:1788574362.006172:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006176:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574362.006178:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574362.006179:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574362.006181:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393128832, offset 0
00000400:00000200:0.0:1788574362.006184:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574362.006186:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574362.006188:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574362.006190:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574362.006192:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574362.006194:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574362.006195:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574362.006198:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330262:330262:256:4294967295] 192.168.202.110@tcp LPNI seq info [330219:330219:8:4294967295]
00000400:00000200:0.0:1788574362.006201:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574362.006205:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574362.006208:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574362.006211:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574362.006213:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574362.006215:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788574362.006218:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574362.006220:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788574362.006223:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574362.006224:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574362.006226:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574362.006228:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574362.006230:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574362.006231:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574362.006233:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574362.006235:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006241:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574362.006243:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006247:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.006248:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574362.006250:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574362.006252:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574362.006254:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574362.006256:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305965606
00000100:00000001:0.0:1788574362.006258:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305966521
00000100:00000001:0.0:1788574362.006259:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305965606
00000100:00000001:0.0:1788574362.006261:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305966521
00000100:00000001:0.0:1788574362.006262:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574362.006264:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.006265:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574362.006267:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574362/real 1788574362]  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574362.006272:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574362.006273:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574362.006274:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574362.006276:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574362.006277:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574362.006278:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574362.006280:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574362.006285:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006289:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574362.006290:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006291:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574362.006292:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574362.006294:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006298:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574362.006301:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006305:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574362.006306:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006307:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574362.006308:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006309:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574362.006310:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574362.006312:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574362.006316:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574362.006318:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574362.006322:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574362.006331:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393128832:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574362.006335:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574362.006338:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006345:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574362.006346:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574362.006349:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574362.006351:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574362.006353:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574362.006356:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393128960:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574362.006360:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574362.006362:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574362.006363:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574362.006365:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574362.006369:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574362.006371:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574362.006374:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574362.006377:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574362.006380:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574362.006383:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393128960, portal 4
00000100:00000001:0.0:1788574362.006385:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574362.006387:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:0.0:1788574362.006422:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006429:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574362.006431:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574362.006434:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574362.006436:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393128960, offset 0
00000400:00000200:0.0:1788574362.006438:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574362.006441:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574362.006444:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574362.006446:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574362.006448:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574362.006450:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574362.006451:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574362.006454:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330263:330263:256:4294967295] 192.168.202.110@tcp LPNI seq info [330220:330220:8:4294967295]
00000400:00000200:0.0:1788574362.006457:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574362.006461:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574362.006464:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574362.006466:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574362.006469:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574362.006471:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788574362.006474:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574362.006476:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788574362.006478:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574362.006480:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574362.006482:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574362.006484:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574362.006486:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574362.006488:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574362.006489:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574362.006492:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006497:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574362.006499:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006503:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.006505:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574362.006507:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574362.006509:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574362.006511:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574362.006512:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305965753
00000100:00000001:0.0:1788574362.006514:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305966521
00000100:00000001:0.0:1788574362.006516:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305965753
00000100:00000001:0.0:1788574362.006517:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305966521
00000100:00000001:0.0:1788574362.006519:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574362.006520:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.006521:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574362.006524:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574362/real 1788574362]  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574362.006528:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574362.006529:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574362.006530:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574362.006531:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574362.006532:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574362.006533:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574362.006535:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574362.006541:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006545:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574362.006546:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006547:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574362.006548:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574362.006550:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006554:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574362.006556:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006561:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574362.006563:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006564:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574362.006565:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006566:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574362.006567:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574362.006569:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574362.006571:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574362.006572:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574362.006575:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574362.006579:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393128960:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574362.006581:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006582:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574362.006584:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393128832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006589:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574362.006590:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574362.006591:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788574362.006593:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574362.006594:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574362.006595:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574362.006597:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574362.006598:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788574362.006599:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574362.006600:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574362.006601:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:0.0:1788574362.006603:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574362.006604:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006606:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574362.006608:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393128960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574362.006612:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574362.006613:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574362.006614:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574362.006615:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574362.006616:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574362.006617:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574362.006618:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574362.006619:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:0.0:1788574362.006620:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574362.006620:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574362.006621:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:0.0:1788574362.006623:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574362.006624:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006625:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.006627:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574362.006628:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.006629:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574362.006630:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574362.517543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574362.517550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574362.517553:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574362.517559:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574362.517563:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574362.517567:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.517570:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574362.517572:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574362.517576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574362.965497:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305966521, at 4305966528
00000100:00000040:1.0:1788574362.965500:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305966521, at 4305966528
00000100:00000010:0.0:1788574362.965512:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574362.965515:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7a80.
00000100:00000001:0.0:1788574363.541488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574363.541494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574363.541496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574363.541501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574363.541504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574363.541508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574363.541510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574363.541511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574363.541514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574364.437577:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574364.437582:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574364.437585:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788574364.565490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574364.565499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574364.565501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574364.565506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574364.565510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574364.565514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574364.565517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574364.565520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574364.565523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574365.589555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574365.589563:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574365.589566:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574365.589572:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574365.589576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574365.589580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574365.589583:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574365.589586:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574365.589589:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574367.125515:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574367.125539:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574367.125548:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:0.0:1788574367.125706:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574367.125708:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574367.125739:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574367.125747:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574367.125754:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788574367.126007:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574367.126012:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574367.126461:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574367.126462:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574367.126658:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574367.126660:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:0.0:1788574367.509566:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574367.509569:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574367.509572:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574368.085467:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305971641, at 4305971648
00000100:00000040:1.0:1788574368.085474:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305971641, at 4305971648
00000100:00000010:0.0:1788574368.085481:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000100:00000010:1.0:1788574368.085484:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00000020:00000001:0.0:1788574370.581561:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574370.581565:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574370.581570:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574372.246049:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574372.246054:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574372.246575:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574372.246577:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574372.246850:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574372.246852:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:1.0:1788574373.205530:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305976761, at 4305976768
00000100:00000040:0.0:1788574373.205531:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305976761, at 4305976768
00000100:00000010:1.0:1788574373.205553:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000100:00000010:0.0:1788574373.205556:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000020:00000001:0.0:1788574373.653649:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574373.653654:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574373.653657:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574376.725590:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574376.725595:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574376.725598:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574377.365563:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574377.365581:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574377.365585:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574377.365591:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574377.365596:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574377.365605:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11308
00000400:00000200:0.0:1788574377.365618:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574377.365626:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574377.365630:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574377.365634:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.365638:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574377.365641:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.365644:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574377.365649:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574377.365652:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574377.365655:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574377.365659:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574377.365662:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574377.365665:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574377.365668:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.365670:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574377.365673:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.365675:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574377.365678:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574377.365682:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574377.365685:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574377.365688:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574377.365692:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574377.365695:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574377.365697:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574377.365708:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574377.365723:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788574377.365726:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574377.365730:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574377.365731:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574377.365734:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574377.365738:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574377.365743:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574377.365751:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574377.365758:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574377.365768:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393129856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574377.365778:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574377.365782:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574377.365786:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393129856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574377.365807:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574377.365810:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574377.365813:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574377.365814:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574377.365815:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574377.365818:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574377.365820:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11308
00000400:00000200:0.0:1788574377.365824:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574377.365937:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574377.365944:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574377.365949:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.365954:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574377.365957:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.365963:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574377.365966:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574377.365968:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574377.365975:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574377.365981:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574377.365990:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574377.365998:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574377.366012:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.366020:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574377.366027:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.366031:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574377.366038:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574377.366043:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574377.366051:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574377.366053:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574377.366063:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574377.366069:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574377.366075:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574377.366083:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574377.366093:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574377.366098:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574377.366108:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788574377.366116:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574377.366125:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574377.366130:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574377.366137:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574377.366144:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574377.366152:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574377.366163:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574377.366173:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574377.366185:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393129984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574377.366218:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574377.366227:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574377.366227:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788574377.366245:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393129984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574377.366255:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574377.366274:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574377.366285:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574377.366290:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574377.366294:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574377.366299:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574377.366307:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11308
00000400:00000200:0.0:1788574377.366322:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574377.366334:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574377.366341:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574377.366351:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.366360:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574377.366371:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.366380:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574377.366455:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574377.366462:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574377.366474:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574377.366484:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574377.366493:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574377.366501:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574377.366514:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.366523:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574377.366529:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574377.366537:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574377.366550:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574377.366562:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574377.366573:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574377.366582:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574377.366593:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574377.366602:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574377.366609:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574377.366616:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574377.366629:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574377.366640:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574377.366655:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788574377.366666:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574377.366676:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574377.366687:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574377.366689:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574377.366699:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574377.366708:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574377.366716:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574377.366733:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574377.366746:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574377.366767:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393130112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574377.366793:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574377.366800:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574377.366807:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393130112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574377.366839:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574377.366843:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574377.366849:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574377.366857:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11318)
00000800:00000200:2.1:1788574377.367008:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788574378.325537:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305981881, at 4305981888
00000100:00000040:1.0:1788574378.325537:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305981881, at 4305981888
00000100:00000010:1.0:1788574378.325557:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000100:00000010:0.0:1788574378.325560:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00000020:00000001:0.0:1788574379.797535:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574379.797539:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574379.797541:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574382.485468:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574382.485478:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574382.485480:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574382.485484:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574382.485488:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574382.485493:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11313
00000400:00000200:0.0:1788574382.485502:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574382.485509:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574382.485512:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574382.485514:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485517:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574382.485521:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485524:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574382.485529:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574382.485532:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574382.485535:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574382.485538:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574382.485541:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574382.485544:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574382.485548:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485550:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574382.485553:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485556:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574382.485560:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574382.485563:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485569:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574382.485572:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574382.485575:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574382.485577:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574382.485579:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574382.485581:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574382.485587:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574382.485589:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574382.485592:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574382.485594:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574382.485596:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574382.485598:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574382.485602:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574382.485608:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574382.485614:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574382.485621:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393130240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574382.485629:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574382.485632:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574382.485636:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393130240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574382.485665:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574382.485669:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574382.485673:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574382.485675:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574382.485676:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574382.485680:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574382.485683:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11313
00000400:00000200:0.0:1788574382.485688:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574382.485692:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574382.485694:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574382.485698:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485702:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574382.485705:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485707:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574382.485710:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574382.485713:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574382.485717:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574382.485719:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574382.485723:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574382.485725:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574382.485729:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485731:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574382.485735:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485739:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574382.485744:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574382.485750:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485756:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574382.485761:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574382.485769:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574382.485773:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574382.485778:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574382.485785:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574382.485791:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574382.485797:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574382.485804:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788574382.485808:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574382.485812:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574382.485814:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574382.485816:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574382.485820:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574382.485824:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574382.485830:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574382.485837:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574382.485847:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393130368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574382.485861:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574382.485866:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574382.485871:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393130368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574382.485898:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574382.485902:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574382.485907:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574382.485911:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574382.485916:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574382.485919:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574382.485922:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11313
00000400:00000200:0.0:1788574382.485931:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574382.485940:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574382.485945:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574382.485950:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485955:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574382.485962:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.485967:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574382.485971:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574382.485974:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574382.485978:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574382.485980:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574382.485987:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574382.485992:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574382.485997:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.486002:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574382.486008:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574382.486013:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574382.486016:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574382.486022:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574382.486029:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574382.486031:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574382.486036:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574382.486040:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574382.486044:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574382.486049:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574382.486053:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574382.486058:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574382.486066:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788574382.486072:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574382.486078:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574382.486081:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574382.486084:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574382.486086:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574382.486089:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574382.486095:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788574382.486102:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574382.486113:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393130496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574382.486126:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574382.486132:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574382.486142:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393130496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788574382.486157:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574382.486158:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574382.486163:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.1:1788574382.486165:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:0.0:1788574382.486170:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574382.486181:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11323)
00000800:00000200:2.1:1788574382.486682:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574382.486684:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788574382.487024:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574382.487028:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1788574382.869621:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574382.869625:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574382.869629:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574383.445517:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305987001, at 4305987008
00000100:00000040:1.0:1788574383.445517:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305987001, at 4305987008
00000100:00000010:0.0:1788574383.445533:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00000100:00000010:1.0:1788574383.445538:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000020:00000001:0.0:1788574385.941541:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574385.941544:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574385.941547:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788574387.605727:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574387.605734:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574387.605739:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574387.605751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.605769:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574387.605774:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574387.605781:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574387.605786:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574387.605791:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574387.605800:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393130624:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574387.605810:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574387.605816:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574387.605820:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574387.605826:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574387.605832:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574387.605837:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574387.605848:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574387.605854:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574387.605866:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574387.605876:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393130624, portal 25
00000100:00000001:0.0:1788574387.605881:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574387.605885:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530777728 : -130484178773888 : ffff895348e01880)
00000100:00000040:0.0:1788574387.605897:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.605949:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574387.605953:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574387.605957:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574387.605962:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393130624, offset 0
00000400:00000200:0.0:1788574387.605974:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574387.605985:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574387.605995:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574387.606001:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574387.606007:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574387.606011:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574387.606016:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574387.606023:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330276:330276:256:4294967295] 192.168.202.110@tcp LPNI seq info [330233:330233:8:4294967295]
00000400:00000200:0.0:1788574387.606031:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574387.606046:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574387.606057:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574387.606063:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574387.606071:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574387.606077:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574387.606085:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574387.606090:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574387.606097:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574387.606101:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574387.606107:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574387.606114:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574387.606119:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574387.606122:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574387.606127:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574387.606134:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606151:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574387.606160:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606200:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574387.606202:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574387.606209:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574387.606217:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574387.606221:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305992121
00000100:00000010:0.0:1788574387.606230:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574387.606238:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574387.606245:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305992121
00000100:00000010:0.0:1788574387.606249:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7900.
00000100:00000001:0.0:1788574387.606251:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574387.606254:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574387.606256:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574387.606259:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574387.606264:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574387/real 1788574387]  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574387.606271:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574387.606272:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574387.606274:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574387.606276:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574387.606277:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574387.606279:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574387.606281:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574387.606287:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606292:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574387.606293:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606295:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574387.606296:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574387.606298:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606304:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574387.606307:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606311:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574387.606313:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606314:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574387.606315:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606317:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574387.606318:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574387.606320:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574387.606323:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574387.606324:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574387.606327:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574387.606333:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393130624:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574387.606336:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606337:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574387.606339:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01880 x1875444393130624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606343:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574387.606345:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574387.606347:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574387.606350:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574387.606351:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574387.606352:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574387.606354:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574387.606356:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788574387.606358:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574387.606360:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574387.606361:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01880.
00000100:00000001:0.0:1788574387.606363:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574387.606364:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606366:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606372:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574387.606373:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574387.606375:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574387.606376:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574387.606376:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574387.606379:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606384:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574387.606385:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574387.606386:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574387.606388:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574387.606389:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574387.606410:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393130752:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574387.606412:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574387.606413:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574387.606414:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574387.606416:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574387.606418:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574387.606419:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574387.606422:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574387.606424:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574387.606426:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574387.606428:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393130752, portal 10
00000100:00000001:0.0:1788574387.606429:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574387.606430:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:0.0:1788574387.606435:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606440:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574387.606441:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574387.606443:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574387.606445:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393130752, offset 0
00000400:00000200:0.0:1788574387.606447:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574387.606450:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574387.606453:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574387.606455:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574387.606457:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574387.606459:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574387.606461:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574387.606470:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330277:330277:256:4294967295] 192.168.202.110@tcp LPNI seq info [330234:330234:8:4294967295]
00000400:00000200:0.0:1788574387.606473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574387.606478:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574387.606481:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574387.606483:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574387.606486:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574387.606488:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574387.606491:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574387.606493:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574387.606495:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574387.606497:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574387.606499:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574387.606501:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574387.606503:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574387.606505:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574387.606506:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574387.606509:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606514:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574387.606516:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606523:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574387.606525:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574387.606528:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574387.606532:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574387.606535:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574387.606544:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305991207
00000100:00000001:0.0:1788574387.606548:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305992121
00000100:00000001:0.0:1788574387.606550:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305991207
00000100:00000001:0.0:1788574387.606552:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305992121
00000100:00000001:0.0:1788574387.606555:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574387.606557:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574387.606559:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574387.606564:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574387/real 1788574387]  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574387.606573:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574387.606575:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574387.606577:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574387.606579:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574387.606580:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574387.606582:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574387.606584:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574387.606590:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606594:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574387.606595:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606597:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574387.606597:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574387.606600:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606604:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574387.606606:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606611:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574387.606613:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606614:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574387.606615:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606616:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574387.606618:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574387.606619:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574387.606621:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574387.606623:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574387.606625:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574387.606630:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393130752:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574387.606633:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574387.606635:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606639:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574387.606640:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574387.606642:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574387.606643:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574387.606644:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574387.606646:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393130880:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574387.606648:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574387.606649:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574387.606650:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574387.606652:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574387.606654:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574387.606656:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574387.606658:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574387.606660:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574387.606662:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574387.606664:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393130880, portal 4
00000100:00000001:0.0:1788574387.606666:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574387.606667:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:0.0:1788574387.606671:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606675:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574387.606676:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574387.606678:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574387.606679:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393130880, offset 0
00000400:00000200:0.0:1788574387.606682:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574387.606684:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574387.606687:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574387.606689:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574387.606691:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574387.606693:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574387.606694:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574387.606696:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330278:330278:256:4294967295] 192.168.202.110@tcp LPNI seq info [330235:330235:8:4294967295]
00000400:00000200:0.0:1788574387.606699:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574387.606703:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574387.606706:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574387.606708:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574387.606711:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574387.606713:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574387.606716:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574387.606717:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574387.606720:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574387.606721:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574387.606724:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574387.606726:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574387.606728:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574387.606729:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574387.606731:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574387.606733:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606738:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574387.606740:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606745:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574387.606746:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574387.606747:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574387.606749:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574387.606751:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574387.606752:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305991353
00000100:00000001:0.0:1788574387.606754:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305992121
00000100:00000001:0.0:1788574387.606756:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305991353
00000100:00000001:0.0:1788574387.606757:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305992121
00000100:00000001:0.0:1788574387.606758:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574387.606760:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574387.606761:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574387.606763:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574387/real 1788574387]  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574387.606767:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574387.606768:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574387.606769:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574387.606770:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574387.606771:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574387.606772:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574387.606774:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574387.606779:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606787:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574387.606788:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606789:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574387.606790:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574387.606793:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606797:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574387.606799:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606803:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574387.606804:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606805:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574387.606807:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606808:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574387.606810:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574387.606812:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574387.606815:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574387.606818:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574387.606822:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574387.606831:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393130880:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574387.606835:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606837:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574387.606841:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393130752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606847:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574387.606849:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574387.606852:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574387.606855:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574387.606856:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574387.606858:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574387.606860:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574387.606862:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788574387.606864:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574387.606865:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574387.606867:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:0.0:1788574387.606870:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574387.606871:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606873:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574387.606876:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393130880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574387.606883:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574387.606885:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574387.606887:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574387.606889:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574387.606891:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574387.606892:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574387.606894:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574387.606896:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788574387.606899:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574387.606900:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574387.606901:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:0.0:1788574387.606904:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574387.606905:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606907:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574387.606911:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574387.606913:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574387.606915:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574387.606916:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574388.565498:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305992121, at 4305992128
00000100:00000040:1.0:1788574388.565498:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305992121, at 4305992128
00000100:00000010:1.0:1788574388.565512:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7900.
00000100:00000010:0.0:1788574388.565515:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788574389.013577:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574389.013580:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574389.013584:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574392.085576:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574392.085591:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574392.085596:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788574392.725764:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574392.725771:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574392.725777:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574392.725791:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.725814:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574392.725828:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574392.725837:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574392.725842:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574392.725849:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574392.725861:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393131008:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574392.725873:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574392.725882:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574392.725888:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574392.725894:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574392.725903:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574392.725907:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574392.725917:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574392.725926:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574392.725937:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574392.725950:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393131008, portal 25
00000100:00000001:0.0:1788574392.725958:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574392.725963:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:0.0:1788574392.725975:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574417 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726011:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574392.726019:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574392.726028:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574392.726034:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393131008, offset 0
00000400:00000200:0.0:1788574392.726052:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574392.726070:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574392.726083:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574392.726093:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574392.726099:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574392.726106:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574392.726111:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574392.726116:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330279:330279:256:4294967295] 192.168.202.110@tcp LPNI seq info [330236:330236:8:4294967295]
00000400:00000200:0.0:1788574392.726131:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574392.726143:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574392.726151:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574392.726158:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574392.726166:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574392.726179:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574392.726191:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574392.726195:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574392.726201:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574392.726204:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574392.726214:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574392.726220:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574392.726228:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574392.726231:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574392.726237:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574392.726242:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574417 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726254:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574392.726258:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574417 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726268:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574392.726270:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574392.726277:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574392.726282:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574392.726286:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4305997241
00000100:00000010:0.0:1788574392.726292:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574392.726302:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574392.726309:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4305997241
00000100:00000010:0.0:1788574392.726314:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7f00.
00000100:00000001:0.0:1788574392.726317:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574392.726322:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574392.726327:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574392.726330:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574392.726343:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574392/real 1788574392]  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574392.726357:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574392.726360:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574392.726362:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574392.726366:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574392.726368:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574392.726371:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574392.726375:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574392.726386:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726431:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574392.726441:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.726445:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574392.726447:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574392.726453:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726462:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574392.726467:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726476:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574392.726480:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.726483:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574392.726484:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.726487:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574392.726490:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574392.726494:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574392.726498:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574392.726501:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574392.726507:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574392.726517:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393131008:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574392.726524:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.726527:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574392.726531:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393131008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726539:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574392.726542:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574392.726546:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574392.726550:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574392.726553:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574392.726555:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574392.726558:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574392.726561:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788574392.726565:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574392.726568:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574392.726570:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:0.0:1788574392.726574:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574392.726576:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.726579:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.726588:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574392.726590:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574392.726593:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574392.726595:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574392.726597:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574392.726602:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726610:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574392.726612:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574392.726615:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574392.726617:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574392.726620:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574392.726624:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393131136:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574392.726627:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574392.726630:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574392.726632:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574392.726634:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574392.726638:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574392.726640:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574392.726645:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574392.726648:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574392.726653:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574392.726656:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393131136, portal 10
00000100:00000001:0.0:1788574392.726659:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574392.726660:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:0.0:1788574392.726668:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574417 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726676:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574392.726679:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574392.726681:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574392.726684:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393131136, offset 0
00000400:00000200:0.0:1788574392.726689:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574392.726693:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574392.726698:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574392.726701:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574392.726704:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574392.726707:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574392.726711:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574392.726715:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330280:330280:256:4294967295] 192.168.202.110@tcp LPNI seq info [330237:330237:8:4294967295]
00000400:00000200:0.0:1788574392.726720:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574392.726728:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574392.726733:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574392.726737:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574392.726742:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574392.726746:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574392.726751:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574392.726754:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574392.726759:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574392.726761:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574392.726764:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574392.726768:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574392.726772:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574392.726774:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574392.726778:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574392.726782:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574417 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726791:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574392.726795:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574417 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726803:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574392.726806:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574392.726809:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574392.726813:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574392.726817:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574392.726820:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305996327
00000100:00000001:0.0:1788574392.726823:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305997241
00000100:00000001:0.0:1788574392.726826:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305996327
00000100:00000001:0.0:1788574392.726828:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305997241
00000100:00000001:0.0:1788574392.726831:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574392.726834:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574392.726837:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574392.726841:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574392/real 1788574392]  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574392.726850:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574392.726852:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574392.726854:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574392.726856:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574392.726858:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574392.726861:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574392.726864:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574392.726874:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726881:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574392.726883:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.726886:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574392.726887:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574392.726892:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726899:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574392.726904:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726913:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574392.726916:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.726918:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574392.726920:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.726922:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574392.726925:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574392.726930:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574392.726933:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574392.726936:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574392.726941:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574392.726951:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393131136:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574392.726956:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574392.726960:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.726968:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574392.726969:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574392.726972:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574392.726974:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574392.726977:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574392.726981:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393131264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574392.726998:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574392.727001:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574392.727003:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574392.727005:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574392.727010:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574392.727012:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574392.727017:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574392.727020:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574392.727024:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574392.727027:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393131264, portal 4
00000100:00000001:0.0:1788574392.727029:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574392.727031:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:0.0:1788574392.727038:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574417 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.727054:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574392.727057:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574392.727059:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574392.727062:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393131264, offset 0
00000400:00000200:0.0:1788574392.727067:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574392.727071:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574392.727076:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574392.727079:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574392.727082:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574392.727085:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574392.727087:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574392.727091:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330281:330281:256:4294967295] 192.168.202.110@tcp LPNI seq info [330238:330238:8:4294967295]
00000400:00000200:0.0:1788574392.727096:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574392.727103:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574392.727109:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574392.727113:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574392.727117:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574392.727121:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574392.727125:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574392.727129:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574392.727134:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574392.727136:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574392.727140:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574392.727144:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574392.727147:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574392.727149:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574392.727153:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574392.727156:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574417 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.727166:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574392.727169:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574417 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.727184:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574392.727186:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574392.727189:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574392.727192:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574392.727197:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574392.727199:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4305996473
00000100:00000001:0.0:1788574392.727202:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4305997241
00000100:00000001:0.0:1788574392.727205:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4305996473
00000100:00000001:0.0:1788574392.727207:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4305997241
00000100:00000001:0.0:1788574392.727210:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574392.727212:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574392.727215:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574392.727220:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574392/real 1788574392]  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574392.727228:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574392.727230:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574392.727232:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574392.727235:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574392.727237:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574392.727240:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574392.727243:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574392.727254:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.727261:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574392.727263:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.727265:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574392.727267:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574392.727272:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.727281:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574392.727286:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.727295:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574392.727297:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.727300:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574392.727301:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.727303:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574392.727306:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574392.727310:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574392.727313:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574392.727316:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574392.727321:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574392.727330:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393131264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574392.727335:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.727337:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574392.727341:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393131136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.727349:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574392.727351:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574392.727353:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574392.727357:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574392.727358:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574392.727360:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574392.727363:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574392.727364:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788574392.727368:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574392.727370:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574392.727371:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:0.0:1788574392.727374:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574392.727376:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.727378:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574392.727382:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393131264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574417 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574392.727408:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574392.727410:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574392.727413:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574392.727416:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574392.727417:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574392.727419:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574392.727421:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574392.727423:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788574392.727426:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574392.727428:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574392.727430:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:0.0:1788574392.727433:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574392.727435:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.727437:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574392.727441:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574392.727442:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574392.727445:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574392.727447:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574393.685498:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4305997241, at 4305997248
00000100:00000040:1.0:1788574393.685498:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4305997241, at 4305997248
00000100:00000010:0.0:1788574393.685513:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574393.685515:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f00.
00000020:00000001:0.0:1788574395.157549:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574395.157552:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574395.157555:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574397.845563:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574397.845579:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574397.845583:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574397.845589:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574397.845595:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574397.845604:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11328
00000400:00000200:0.0:1788574397.845616:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574397.845624:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574397.845627:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574397.845630:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845633:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574397.845636:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845638:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574397.845641:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574397.845643:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574397.845646:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574397.845649:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574397.845651:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574397.845653:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574397.845655:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845656:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574397.845658:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845659:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574397.845662:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574397.845664:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845666:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574397.845669:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574397.845672:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574397.845674:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574397.845676:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574397.845679:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574397.845689:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788574397.845692:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574397.845696:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574397.845697:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574397.845700:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574397.845703:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574397.845708:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574397.845725:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574397.845732:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574397.845741:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393131392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574397.845752:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574397.845758:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574397.845763:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393131392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574397.845793:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574397.845797:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574397.845802:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574397.845803:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574397.845805:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574397.845808:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574397.845811:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11328
00000400:00000200:0.0:1788574397.845815:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574397.845817:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574397.845820:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574397.845825:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845828:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574397.845832:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845835:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574397.845838:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574397.845840:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574397.845841:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574397.845845:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574397.845850:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574397.845854:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574397.845860:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845866:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574397.845869:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845873:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574397.845879:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574397.845885:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574397.845889:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574397.845894:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574397.845901:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574397.845906:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574397.845910:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574397.845915:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574397.845919:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574397.845924:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574397.845931:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574397.845934:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574397.845938:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574397.845941:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574397.845943:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574397.845946:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574397.845948:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574397.845953:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574397.845959:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574397.845972:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393131520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574397.845984:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574397.845988:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574397.845999:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393131520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574397.846019:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574397.846023:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574397.846030:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574397.846034:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574397.846055:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574397.846060:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574397.846065:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11328
00000400:00000200:0.0:1788574397.846070:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574397.846077:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574397.846080:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574397.846086:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.846092:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574397.846100:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.846108:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574397.846114:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574397.846118:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574397.846123:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574397.846125:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574397.846131:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574397.846134:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574397.846141:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.846146:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574397.846151:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574397.846153:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574397.846155:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574397.846160:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574397.846162:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574397.846167:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574397.846172:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574397.846179:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574397.846182:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574397.846185:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:2.1:1788574397.846187:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000008:00100000:0.0:1788574397.846189:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574397.846197:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574397.846201:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574397.846206:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574397.846212:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574397.846218:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574397.846221:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574397.846224:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574397.846227:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574397.846228:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574397.846233:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574397.846238:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574397.846246:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393131648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574397.846267:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574397.846272:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574397.846283:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393131648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574397.846302:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574397.846307:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574397.846313:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574397.846321:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11339)
00000800:00000200:2.1:1788574397.846625:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574397.846626:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788574397.846836:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574397.846837:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:0.0:1788574398.229613:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574398.229617:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574398.229621:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574398.805504:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306002361, at 4306002368
00000100:00000040:0.0:1788574398.805513:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306002361, at 4306002368
00000100:00000010:1.0:1788574398.805528:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000100:00000010:0.0:1788574398.805536:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000020:00000001:0.0:1788574401.301605:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574401.301612:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574401.301617:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574402.965499:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574402.965509:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574402.965512:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574402.965517:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574402.965521:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574402.965528:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11334
00000400:00000200:0.0:1788574402.965538:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574402.965543:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574402.965546:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574402.965548:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965551:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574402.965552:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965555:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574402.965558:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574402.965559:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574402.965561:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574402.965564:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574402.965566:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574402.965567:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574402.965569:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965571:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574402.965572:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965573:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574402.965575:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574402.965577:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965579:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574402.965581:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574402.965584:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574402.965586:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574402.965588:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574402.965590:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574402.965597:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788574402.965600:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574402.965603:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574402.965605:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574402.965607:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574402.965609:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574402.965613:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574402.965619:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574402.965624:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574402.965631:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393131776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574402.965639:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574402.965642:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574402.965645:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393131776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574402.965670:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574402.965673:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574402.965676:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574402.965677:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574402.965678:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574402.965681:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574402.965683:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11334
00000400:00000200:0.0:1788574402.965685:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574402.965688:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574402.965690:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574402.965692:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965693:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574402.965695:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965697:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574402.965698:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574402.965700:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574402.965701:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574402.965703:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574402.965705:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574402.965707:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574402.965710:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965711:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574402.965713:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965714:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574402.965716:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574402.965717:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965719:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574402.965720:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574402.965725:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574402.965732:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574402.965735:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574402.965746:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574402.965750:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574402.965752:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574402.965755:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788574402.965759:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574402.965762:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574402.965765:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574402.965774:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574402.965779:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574402.965784:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574402.965792:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788574402.965798:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574402.965809:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393131904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574402.965833:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574402.965839:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574402.965861:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393131904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574402.965880:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574402.965884:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574402.965889:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574402.965893:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574402.965899:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574402.965900:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574402.965905:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11334
00000400:00000200:0.0:1788574402.965912:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574402.965923:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574402.965931:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574402.965935:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965936:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574402.965938:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965939:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574402.965942:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574402.965947:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574402.965949:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574402.965951:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574402.965953:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574402.965960:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574402.965972:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965975:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574402.965982:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574402.965992:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574402.966004:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574402.966010:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574402.966022:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574402.966026:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574402.966029:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574402.966036:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574402.966038:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574402.966043:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574402.966048:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574402.966050:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574402.966057:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574402.966067:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574402.966069:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574402.966071:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574402.966073:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574402.966079:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574402.966099:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574402.966106:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574402.966111:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574402.966127:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393132032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574402.966141:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574402.966149:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574402.966160:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393132032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574402.966186:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574402.966192:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574402.966204:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574402.966217:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11344)
00000800:00000200:2.1:1788574402.966269:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574402.966272:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788574402.966708:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574402.966711:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788574402.966958:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574402.966960:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:0.0:1788574403.925537:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306007481, at 4306007488
00000100:00000040:1.0:1788574403.925537:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306007481, at 4306007488
00000100:00000010:1.0:1788574403.925556:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000100:00000010:0.0:1788574403.925559:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00000020:00000001:0.0:1788574404.373637:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574404.373643:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574404.373647:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000001:1.0:1788574406.037771:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574406.037772:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574406.037798:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574406.037804:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574406.037807:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1788574407.445530:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574407.445534:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574407.445537:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574408.086188:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574408.086192:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788574408.086589:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574408.086591:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788574408.086836:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574408.086838:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000040:0.0:1788574409.045477:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306012601, at 4306012608
00000100:00000040:1.0:1788574409.045477:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306012601, at 4306012608
00000100:00000010:1.0:1788574409.045491:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000100:00000010:0.0:1788574409.045493:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000020:00000001:0.0:1788574410.517591:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574410.517595:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574410.517598:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574413.206121:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574413.206127:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574413.206647:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574413.206650:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574413.206908:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574413.206910:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:0.0:1788574413.589693:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574413.589703:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574413.589707:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574414.165500:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306017721, at 4306017728
00000100:00000040:1.0:1788574414.165500:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306017721, at 4306017728
00000100:00000010:1.0:1788574414.165513:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00000100:00000010:0.0:1788574414.165516:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000020:00000001:0.0:1788574416.661578:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574416.661583:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574416.661586:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788574418.325928:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574418.325932:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574418.326306:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574418.326308:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574418.326522:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574418.326523:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:1.0:1788574419.285507:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306022841, at 4306022848
00000100:00000040:0.0:1788574419.285508:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306022841, at 4306022848
00000100:00000010:1.0:1788574419.285524:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000100:00000010:0.0:1788574419.285526:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000020:00000001:0.0:1788574419.733551:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574419.733554:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574419.733557:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574422.805586:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574422.805590:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574422.805593:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574423.445488:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574423.445497:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574423.445499:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574423.445502:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574423.445506:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574423.445512:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11354
00000400:00000200:0.0:1788574423.445520:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574423.445525:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574423.445528:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574423.445530:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445533:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574423.445534:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445536:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574423.445539:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574423.445541:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574423.445543:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574423.445545:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574423.445547:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574423.445548:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574423.445550:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445551:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574423.445553:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445554:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574423.445557:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574423.445559:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445561:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574423.445564:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574423.445566:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574423.445568:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574423.445570:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574423.445572:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574423.445580:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574423.445583:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574423.445585:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574423.445586:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574423.445589:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574423.445591:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574423.445595:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574423.445602:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788574423.445608:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574423.445616:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393133312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574423.445623:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574423.445627:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574423.445630:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393133312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574423.445648:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574423.445651:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574423.445654:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574423.445655:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574423.445656:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574423.445658:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574423.445660:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11354
00000400:00000200:0.0:1788574423.445663:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574423.445665:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574423.445667:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574423.445670:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445672:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574423.445674:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445675:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574423.445677:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574423.445678:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574423.445680:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574423.445681:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574423.445686:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574423.445688:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574423.445692:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445694:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574423.445699:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445701:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574423.445705:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574423.445709:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445714:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574423.445717:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574423.445722:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574423.445725:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574423.445730:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574423.445733:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574423.445737:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574423.445741:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574423.445744:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788574423.445748:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574423.445751:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574423.445753:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574423.445756:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574423.445757:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574423.445759:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574423.445763:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788574423.445767:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574423.445773:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393133440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574423.445782:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574423.445784:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574423.445791:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393133440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574423.445799:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574423.445801:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574423.445808:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574423.445810:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574423.445813:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574423.445816:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574423.445819:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11354
00000400:00000200:0.0:1788574423.445823:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574423.445828:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574423.445829:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574423.445833:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445836:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574423.445838:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445840:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574423.445841:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574423.445842:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574423.445845:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574423.445848:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574423.445849:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574423.445851:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574423.445853:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445858:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574423.445863:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445867:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574423.445870:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574423.445873:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574423.445874:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574423.445875:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574423.445877:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574423.445879:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574423.445880:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574423.445881:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574423.445884:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574423.445887:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574423.445892:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788574423.445896:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574423.445900:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574423.445901:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574423.445904:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574423.445906:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574423.445907:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574423.445911:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788574423.445924:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574423.445930:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393133568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574423.445939:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574423.445943:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574423.445950:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393133568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574423.445961:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574423.445964:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574423.445969:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574423.445976:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11364)
00000800:00000200:2.1:1788574423.446015:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574423.446018:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574423.446347:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574423.446348:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574423.446669:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574423.446671:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:0.0:1788574424.405545:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306027961, at 4306027968
00000100:00000040:1.0:1788574424.405545:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306027961, at 4306027968
00000100:00000010:1.0:1788574424.405564:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00000100:00000010:0.0:1788574424.405569:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000020:00000001:0.0:1788574425.877567:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574425.877571:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574425.877575:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574428.565478:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574428.565487:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574428.565489:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574428.565493:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574428.565497:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574428.565502:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11359
00000400:00000200:0.0:1788574428.565510:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574428.565515:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574428.565518:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574428.565533:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565537:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574428.565540:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565543:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574428.565547:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574428.565549:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574428.565553:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574428.565556:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574428.565558:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574428.565561:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574428.565564:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565566:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574428.565568:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565571:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574428.565575:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574428.565578:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565581:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574428.565584:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574428.565588:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574428.565591:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574428.565594:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574428.565597:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574428.565605:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788574428.565608:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574428.565612:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574428.565614:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574428.565617:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574428.565620:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574428.565624:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574428.565631:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574428.565639:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574428.565647:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393133696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574428.565658:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574428.565663:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574428.565667:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393133696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574428.565684:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574428.565687:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574428.565689:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574428.565691:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574428.565692:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574428.565694:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574428.565696:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11359
00000400:00000200:0.0:1788574428.565700:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574428.565702:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574428.565704:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574428.565707:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565711:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574428.565716:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565719:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574428.565722:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574428.565724:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574428.565726:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574428.565729:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574428.565733:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574428.565738:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574428.565743:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565746:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574428.565750:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565753:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574428.565758:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574428.565765:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565770:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574428.565772:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574428.565778:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574428.565783:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574428.565785:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574428.565788:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574428.565794:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574428.565796:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574428.565802:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574428.565806:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574428.565809:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574428.565811:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574428.565816:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574428.565817:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574428.565818:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574428.565823:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788574428.565829:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574428.565837:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393133824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574428.565852:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574428.565857:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574428.565865:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393133824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574428.565876:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574428.565880:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574428.565884:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574428.565885:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574428.565889:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574428.565892:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574428.565896:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11359
00000400:00000200:0.0:1788574428.565901:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574428.565907:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574428.565910:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574428.565916:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565922:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574428.565928:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565934:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574428.565938:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574428.565942:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574428.565943:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574428.565945:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574428.565949:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574428.565954:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574428.565960:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565965:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574428.565969:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565971:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574428.565976:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574428.565978:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574428.565983:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574428.565988:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574428.565991:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574428.565996:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574428.565999:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:2.1:1788574428.566000:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000008:00100000:0.0:1788574428.566002:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:2.1:1788574428.566003:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000008:00000001:0.0:1788574428.566005:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574428.566008:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574428.566013:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788574428.566016:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574428.566019:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574428.566022:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574428.566025:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574428.566028:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574428.566031:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574428.566034:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574428.566038:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574428.566044:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393133952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574428.566067:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574428.566069:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574428.566075:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393133952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574428.566108:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574428.566112:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574428.566116:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574428.566123:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11369)
00000800:00000200:2.1:1788574428.566528:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574428.566531:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788574428.566839:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574428.566842:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1788574428.949543:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574428.949546:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574428.949549:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788574429.525511:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306033081, at 4306033088
00000100:00000040:1.0:1788574429.525519:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306033081, at 4306033088
00000100:00000010:0.0:1788574429.525524:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000100:00000010:1.0:1788574429.525530:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000020:00000001:0.0:1788574432.021560:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574432.021563:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574432.021567:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788574433.685529:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574433.685540:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574433.685543:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574433.685548:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574433.685553:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574433.685561:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11364
00000400:00000200:0.0:1788574433.685568:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574433.685574:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574433.685577:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574433.685580:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685583:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574433.685586:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685588:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574433.685590:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574433.685592:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574433.685594:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574433.685596:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574433.685598:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574433.685600:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574433.685604:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685606:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574433.685608:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685610:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574433.685614:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574433.685617:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685620:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574433.685623:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574433.685627:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574433.685630:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574433.685633:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574433.685635:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574433.685644:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788574433.685648:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574433.685652:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574433.685653:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574433.685656:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574433.685658:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574433.685662:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574433.685668:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574433.685673:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574433.685681:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393134080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574433.685689:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574433.685692:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574433.685695:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393134080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574433.685712:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574433.685714:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574433.685717:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574433.685718:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574433.685719:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574433.685722:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574433.685724:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11364
00000400:00000200:0.0:1788574433.685728:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574433.685731:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574433.685733:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574433.685735:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685740:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574433.685744:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685747:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574433.685751:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574433.685752:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574433.685756:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574433.685759:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574433.685763:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574433.685768:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574433.685774:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685777:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574433.685782:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685788:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574433.685792:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574433.685798:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685801:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574433.685804:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574433.685809:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574433.685815:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574433.685816:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574433.685820:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574433.685826:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574433.685828:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574433.685833:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788574433.685837:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574433.685842:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574433.685843:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574433.685846:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574433.685849:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574433.685851:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574433.685856:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574433.685861:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574433.685869:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393134208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574433.685886:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574433.685891:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574433.685896:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393134208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574433.685902:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574433.685906:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574433.685914:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574433.685915:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574433.685918:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574433.685924:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574433.685927:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11364
00000400:00000200:0.0:1788574433.685933:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574433.685937:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574433.685941:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574433.685948:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685952:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574433.685958:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685962:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574433.685963:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574433.685965:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574433.685967:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574433.685969:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574433.685974:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574433.685978:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574433.685984:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685988:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574433.685991:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574433.685995:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574433.686000:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574433.686006:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574433.686008:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574433.686009:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574433.686011:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574433.686016:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574433.686020:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574433.686023:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574433.686028:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574433.686031:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574433.686036:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788574433.686041:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574433.686044:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574433.686046:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574433.686049:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574433.686051:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574433.686054:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574433.686058:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:0.0:1788574433.686059:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574433.686065:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574433.686065:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00100000:0.0:1788574433.686075:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393134336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574433.686084:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574433.686088:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574433.686095:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393134336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574433.686107:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574433.686110:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574433.686114:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574433.686126:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11374)
00000800:00000200:2.1:1788574433.686432:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574433.686434:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788574433.686671:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574433.686673:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1788574434.645554:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306038201, at 4306038208
00000100:00000040:1.0:1788574434.645554:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306038201, at 4306038208
00000100:00000010:1.0:1788574434.645574:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00000100:00000010:0.0:1788574434.645575:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000020:00000001:0.0:1788574435.093681:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574435.093685:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574435.093687:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1788574438.165629:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574438.165636:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574438.165641:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788574438.805778:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574438.805786:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574438.805789:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574438.805807:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.805831:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574438.805834:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574438.805839:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574438.805842:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574438.805847:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574438.805853:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393134464:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574438.805867:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574438.805873:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574438.805877:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574438.805882:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574438.805891:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788574438.805898:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574438.805913:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574438.805917:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574438.805925:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574438.805942:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393134464, portal 25
00000100:00000001:0.0:1788574438.805952:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574438.805954:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:0.0:1788574438.805961:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.805979:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574438.805984:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574438.805986:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574438.805993:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393134464, offset 0
00000400:00000200:0.0:1788574438.806003:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574438.806014:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574438.806025:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574438.806031:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574438.806037:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574438.806045:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574438.806050:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574438.806056:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330306:330306:256:4294967295] 192.168.202.110@tcp LPNI seq info [330263:330263:8:4294967295]
00000400:00000200:0.0:1788574438.806062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574438.806072:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574438.806083:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574438.806089:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574438.806100:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574438.806112:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574438.806124:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574438.806132:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574438.806149:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574438.806156:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574438.806165:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574438.806180:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574438.806191:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574438.806194:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574438.806198:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574438.806205:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806232:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574438.806243:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806258:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574438.806263:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574438.806269:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574438.806275:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574438.806279:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306043321
00000100:00000010:0.0:1788574438.806285:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574438.806296:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574438.806306:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306043321
00000100:00000010:0.0:1788574438.806320:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7600.
00000100:00000001:0.0:1788574438.806326:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574438.806332:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574438.806335:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574438.806339:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574438.806348:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574438/real 1788574438]  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574438.806370:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574438.806373:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574438.806375:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574438.806379:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574438.806383:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574438.806386:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574438.806424:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574438.806448:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806483:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574438.806489:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.806496:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574438.806501:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574438.806514:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806549:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574438.806560:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806569:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574438.806573:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.806580:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574438.806581:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.806584:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574438.806589:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574438.806605:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574438.806612:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574438.806618:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574438.806622:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574438.806636:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393134464:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574438.806644:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.806652:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574438.806660:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393134464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806692:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574438.806697:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574438.806700:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788574438.806708:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574438.806711:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574438.806717:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574438.806720:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574438.806723:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574438.806731:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574438.806733:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574438.806737:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:0.0:1788574438.806745:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574438.806749:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.806752:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.806764:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574438.806769:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574438.806772:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574438.806779:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574438.806781:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574438.806787:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806794:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574438.806796:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574438.806799:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574438.806801:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574438.806803:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574438.806805:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393134592:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574438.806808:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574438.806810:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574438.806812:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574438.806814:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574438.806817:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788574438.806819:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574438.806822:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574438.806824:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574438.806827:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574438.806829:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393134592, portal 10
00000100:00000001:0.0:1788574438.806830:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574438.806831:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:0.0:1788574438.806836:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806841:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574438.806842:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574438.806844:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574438.806846:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393134592, offset 0
00000400:00000200:0.0:1788574438.806850:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574438.806854:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574438.806858:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574438.806862:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574438.806865:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574438.806867:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574438.806870:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574438.806873:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330307:330307:256:4294967295] 192.168.202.110@tcp LPNI seq info [330264:330264:8:4294967295]
00000400:00000200:0.0:1788574438.806877:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574438.806885:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574438.806891:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574438.806894:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574438.806899:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574438.806902:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574438.806909:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574438.806912:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574438.806917:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574438.806920:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574438.806923:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574438.806927:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574438.806930:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574438.806933:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574438.806936:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574438.806940:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806949:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574438.806952:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.806960:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574438.806962:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574438.806965:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574438.806969:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574438.806973:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574438.806976:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306042407
00000100:00000001:0.0:1788574438.806979:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306043321
00000100:00000001:0.0:1788574438.806982:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306042407
00000100:00000001:0.0:1788574438.806984:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306043321
00000100:00000001:0.0:1788574438.806987:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574438.806989:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574438.806991:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574438.806996:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574438/real 1788574438]  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574438.807007:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574438.807008:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574438.807010:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574438.807013:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574438.807015:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574438.807018:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574438.807021:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574438.807031:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807039:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574438.807041:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574438.807044:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574438.807048:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807056:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574438.807061:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807069:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574438.807072:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807074:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574438.807075:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807077:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574438.807080:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574438.807083:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574438.807086:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574438.807089:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574438.807094:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574438.807104:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393134592:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574438.807108:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574438.807112:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807120:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574438.807122:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574438.807125:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574438.807127:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574438.807129:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574438.807132:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393134720:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574438.807137:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574438.807139:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574438.807141:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574438.807143:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574438.807148:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574438.807151:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574438.807155:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574438.807158:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574438.807162:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574438.807166:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393134720, portal 4
00000100:00000001:0.0:1788574438.807168:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574438.807169:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:0.0:1788574438.807176:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807184:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574438.807186:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574438.807188:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574438.807191:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393134720, offset 0
00000400:00000200:0.0:1788574438.807196:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574438.807199:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574438.807203:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574438.807206:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574438.807209:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574438.807212:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574438.807215:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574438.807218:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330308:330308:256:4294967295] 192.168.202.110@tcp LPNI seq info [330265:330265:8:4294967295]
00000400:00000200:0.0:1788574438.807223:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574438.807229:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574438.807234:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574438.807238:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574438.807242:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574438.807246:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574438.807250:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574438.807253:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574438.807258:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574438.807260:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574438.807263:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574438.807267:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574438.807270:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574438.807272:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574438.807275:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574438.807279:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807288:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574438.807291:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807299:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574438.807301:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574438.807303:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574438.807307:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574438.807310:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574438.807312:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306042553
00000100:00000001:0.0:1788574438.807315:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306043321
00000100:00000001:0.0:1788574438.807317:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306042553
00000100:00000001:0.0:1788574438.807320:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306043321
00000100:00000001:0.0:1788574438.807322:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574438.807325:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574438.807327:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574438.807331:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574438/real 1788574438]  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574438.807340:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574438.807342:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574438.807343:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574438.807346:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574438.807348:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574438.807351:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574438.807354:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574438.807364:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807371:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574438.807373:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807375:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574438.807376:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574438.807380:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807388:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574438.807415:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807424:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574438.807427:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807429:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574438.807431:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807433:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574438.807435:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574438.807438:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574438.807442:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574438.807445:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574438.807449:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574438.807458:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393134720:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574438.807462:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807464:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574438.807481:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393134592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807489:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574438.807491:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574438.807493:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788574438.807496:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574438.807498:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574438.807499:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574438.807502:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574438.807504:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788574438.807507:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574438.807509:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574438.807510:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:0.0:1788574438.807513:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574438.807515:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807517:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574438.807521:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393134720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574438.807536:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574438.807538:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574438.807539:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788574438.807542:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574438.807543:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574438.807544:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574438.807547:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574438.807548:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788574438.807550:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574438.807552:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574438.807553:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:0.0:1788574438.807556:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574438.807557:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807560:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574438.807563:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574438.807565:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574438.807567:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574438.807569:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574439.765530:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306043321, at 4306043328
00000100:00000040:0.0:1788574439.765540:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306043321, at 4306043328
00000100:00000010:1.0:1788574439.765546:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7600.
00000100:00000010:0.0:1788574439.765550:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000020:00000001:0.0:1788574441.237687:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574441.237690:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574441.237693:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.0:1788574443.925513:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574443.925540:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574443.925551:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:0.0:1788574443.925765:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000001:1.0:1788574443.925766:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574443.925772:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000800:00000001:0.0:1788574443.925774:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000001:1.0:1788574443.925777:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574443.925787:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000001:0.0:1788574443.925789:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788574443.925799:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574443.925803:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574443.925806:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574443.925812:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574443.925817:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574443.925821:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574443.925826:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574443.925834:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393134848:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574443.925839:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574443.925843:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574443.925847:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574443.925850:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574443.925859:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574443.925863:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574443.925880:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574443.925888:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574443.925898:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574443.925907:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393134848, portal 25
00000100:00000001:1.0:1788574443.925913:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574443.925916:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:1.0:1788574443.925926:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574443.925935:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574443.925941:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574443.925947:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2110.
00000100:00000200:1.0:1788574443.925951:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393134848, offset 0
00000400:00000200:1.0:1788574443.925965:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574443.925978:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574443.925990:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574443.925999:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574443.926006:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574443.926012:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574443.926019:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574443.926024:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330309:330309:256:4294967295] 192.168.202.110@tcp LPNI seq info [330266:330266:8:4294967295]
00000400:00000200:1.0:1788574443.926028:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574443.926044:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574443.926057:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574443.926063:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ced00.
00000800:00000200:1.0:1788574443.926074:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574443.926080:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574443.926101:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574443.926111:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574443.926118:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306048441
00000100:00000010:1.0:1788574443.926128:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9960.
00000100:00000001:1.0:1788574443.926142:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574443.926153:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306048441
00000100:00000010:1.0:1788574443.926162:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9b40.
00000100:00000001:1.0:1788574443.926170:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574443.926178:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574443.926186:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574443.926194:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574443.926200:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574443.926210:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574443.926213:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1788574443.926217:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574443.926220:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574443.926223:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574443.926228:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574443.926234:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574443.926235:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574443.926237:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574443.926239:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574443.926241:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574443.926243:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393134976:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574443.926246:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574443.926248:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574443.926250:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574443.926251:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574443.926255:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758400.
02000000:00000001:1.0:1788574443.926257:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574443.926261:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574443.926263:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d790.
00000400:00000010:1.0:1788574443.926267:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2550.
00000100:00000200:1.0:1788574443.926270:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393134976, portal 10
00000100:00000001:1.0:1788574443.926271:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574443.926272:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:1.0:1788574443.926278:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574443.926283:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574443.926284:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574443.926286:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2088.
00000100:00000200:1.0:1788574443.926288:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393134976, offset 0
00000400:00000200:1.0:1788574443.926291:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574443.926294:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574443.926297:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574443.926299:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574443.926301:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574443.926303:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574443.926305:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574443.926307:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330310:330310:255:4294967295] 192.168.202.110@tcp LPNI seq info [330267:330267:7:4294967295]
00000400:00000200:1.0:1788574443.926310:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574443.926315:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574443.926318:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574443.926321:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cef00.
00000800:00000200:1.0:1788574443.926324:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574443.926327:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574443.926329:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574443.926332:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574443.926335:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574443.926336:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306047526
00000100:00000001:1.0:1788574443.926339:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306048441
00000100:00000001:1.0:1788574443.926340:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306047526
00000100:00000001:1.0:1788574443.926342:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306048441
00000100:00000001:1.0:1788574443.926343:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574443.926344:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574443.926345:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574443.926348:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574443.926352:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574443.926353:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574443.926355:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574443.926356:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574443.926357:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574443.926359:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574443.926361:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574443.926362:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574443.926363:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574443.926364:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574443.926366:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574443.926368:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574443.926370:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574443.926372:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5df20.
00000400:00000010:1.0:1788574443.926374:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574443.926376:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393135104, portal 4
00000100:00000001:1.0:1788574443.926378:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574443.926379:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:1.0:1788574443.926383:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574443.926387:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574443.926388:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574443.926390:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2660.
00000100:00000200:1.0:1788574443.926419:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393135104, offset 0
00000400:00000200:1.0:1788574443.926428:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574443.926432:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574443.926435:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574443.926438:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574443.926441:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574443.926443:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574443.926446:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574443.926449:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330311:330311:254:4294967295] 192.168.202.110@tcp LPNI seq info [330268:330268:6:4294967295]
00000400:00000200:1.0:1788574443.926453:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574443.926459:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574443.926464:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574443.926473:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce500.
00000800:00000200:1.0:1788574443.926477:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574443.926480:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574443.926483:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574443.926487:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574443.926490:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574443.926493:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306047673
00000100:00000001:1.0:1788574443.926496:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306048441
00000100:00000001:1.0:1788574443.926498:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306047673
00000100:00000001:1.0:1788574443.926501:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306048441
00000100:00000001:1.0:1788574443.926503:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574443.926505:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574443.926508:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574443.926510:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574443.926513:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574443.926514:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1788574443.926516:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574443.926518:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574443.926519:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574443.926522:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574444.309611:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574444.309618:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574444.309623:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788574444.885558:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306048441, at 4306048448
00000100:00000040:0.0:1788574444.885562:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306048441, at 4306048448
00000100:00000010:1.0:1788574444.885578:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9b40.
00000100:00000010:0.0:1788574444.885582:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9960.
00000400:00000100:0.0:1788574446.997745:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:0.0:1788574446.997800:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1788574446.997810:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574446.997813:0:2350:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:0.0:1788574446.997819:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574446.997823:0:2350:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1788574446.997828:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788574446.997832:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ced00.
00000400:00000200:0.0:1788574446.997839:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788574446.997844:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788574446.997854:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2110
00000400:00000010:0.0:1788574446.997857:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2110.
00000100:00000001:0.0:1788574446.997864:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574446.997875:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574446.997916:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574446.997922:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574446.997932:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574446.997935:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788574446.997950:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788574446.997953:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cef00.
00000400:00000200:0.0:1788574446.997956:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788574446.997960:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788574446.997964:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2088
00000400:00000010:0.0:1788574446.997966:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2088.
00000100:00000001:0.0:1788574446.997969:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574446.997976:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574446.997985:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574446.997988:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574446.998002:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574446.998004:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788574446.998010:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788574446.998013:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce500.
00000400:00000200:0.0:1788574446.998016:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788574446.998019:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788574446.998023:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2660
00000400:00000010:0.0:1788574446.998025:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2660.
00000100:00000001:0.0:1788574446.998027:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574446.998032:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574446.998041:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574446.998044:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574446.998052:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574446.998053:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574446.998149:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574446.998155:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574446.998159:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574446.998169:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574443/real 1788574446]  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574446.998185:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574446.998188:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574446.998192:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d790.
00000400:00000200:1.0:1788574446.998197:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2550
00000400:00000010:1.0:1788574446.998198:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2550.
00000100:00000001:1.0:1788574446.998202:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574446.998206:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574446.998216:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998225:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574446.998228:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998231:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574446.998233:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574446.998238:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998243:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574446.998249:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998255:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574446.998258:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998259:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574446.998260:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998263:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574446.998267:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574446.998271:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574446.998277:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574446.998280:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574446.998284:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574446.998293:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393134976:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574446.998298:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574446.998302:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574443/real 1788574446]  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574446.998311:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574446.998312:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574446.998315:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5df20.
00000400:00000200:1.0:1788574446.998318:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574446.998320:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574446.998323:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574446.998327:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574446.998335:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998342:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574446.998344:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998347:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574446.998349:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574446.998352:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998359:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574446.998363:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998369:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574446.998372:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998373:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574446.998374:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998376:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574446.998378:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574446.998381:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574446.998385:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574446.998387:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574446.998594:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574446.998605:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574446.998610:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574446.998615:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574443/real 1788574446]  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574446.998625:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574446.998627:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574446.998630:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574446.998634:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574446.998636:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574446.998639:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574446.998643:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574446.998651:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998659:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574446.998660:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998663:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574446.998664:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574446.998668:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998675:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574446.998680:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998687:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574446.998690:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998692:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574446.998693:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998695:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574446.998698:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574446.998700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574446.998704:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574446.998707:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574446.998712:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574446.998720:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393134848:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574446.998724:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998728:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574446.998731:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393134976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998738:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574446.998742:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574446.998748:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758400.
02000000:00000001:1.0:1788574446.998753:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574446.998757:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574446.998759:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574446.998762:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574446.998766:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:1.0:1788574446.998771:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574446.998773:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574446.998776:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:1.0:1788574446.998781:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574446.998783:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998784:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574446.998788:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393135104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998795:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574446.998797:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574446.998799:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574446.998802:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574446.998803:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574446.998804:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574446.998807:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574446.998808:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:1.0:1788574446.998811:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574446.998813:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574446.998814:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:1.0:1788574446.998818:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574446.998820:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998821:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574446.998825:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393134848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574446.998833:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574446.998835:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574446.998837:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574446.998840:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574446.998842:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574446.998843:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574446.998846:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574446.998848:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:1.0:1788574446.998851:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574446.998852:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574446.998853:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:1.0:1788574446.998856:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574446.998857:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998860:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574446.998871:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574446.998874:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574446.998877:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574446.998879:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788574447.381625:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788574447.381630:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788574447.381633:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788574449.045767:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574449.045779:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574449.045786:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574449.045807:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.045843:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574449.045848:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574449.045876:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574449.045882:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574449.045889:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574449.045900:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393135232:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574449.045923:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574449.045932:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574449.045939:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574449.045946:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574449.045958:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574449.045966:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574449.045984:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574449.045993:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574449.046007:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574449.046022:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393135232, portal 25
00000100:00000001:1.0:1788574449.046031:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574449.046037:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:1.0:1788574449.046056:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574474 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046084:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574449.046094:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574449.046103:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574449.046113:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393135232, offset 0
00000400:00000200:1.0:1788574449.046131:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574449.046150:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574449.046171:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574449.046190:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574449.046204:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574449.046216:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574449.046225:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574449.046238:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330312:330312:256:4294967295] 192.168.202.110@tcp LPNI seq info [330269:330269:8:4294967295]
00000400:00000200:1.0:1788574449.046259:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574449.046287:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574449.046304:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574449.046317:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574449.046340:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574449.046352:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1788574449.046368:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574449.046377:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1788574449.046388:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574449.046461:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574449.046471:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574449.046480:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574449.046489:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574449.046493:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574449.046499:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574449.046505:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574474 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046526:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574449.046531:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574474 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046540:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574449.046543:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574449.046552:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574449.046556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574449.046559:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306053562
00000100:00000010:1.0:1788574449.046566:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9b40.
00000100:00000001:1.0:1788574449.046576:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574449.046597:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306053562
00000100:00000010:1.0:1788574449.046602:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9240.
00000100:00000001:1.0:1788574449.046606:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574449.046611:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574449.046615:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574449.046619:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574449.046626:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574449/real 1788574449]  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574449.046638:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574449.046640:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574449.046644:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574449.046648:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574449.046650:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574449.046654:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574449.046658:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574449.046667:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046673:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574449.046675:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.046678:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574449.046680:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574449.046683:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046689:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574449.046694:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046700:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574449.046702:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.046703:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574449.046704:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.046706:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574449.046708:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574449.046711:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574449.046715:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574449.046718:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574449.046723:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574449.046732:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393135232:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574449.046742:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.046744:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574449.046748:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393135232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046756:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574449.046760:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574449.046763:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574449.046767:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574449.046771:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574449.046773:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574449.046776:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574449.046779:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:1.0:1788574449.046783:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574449.046785:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574449.046788:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:1.0:1788574449.046792:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574449.046793:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.046796:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.046806:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574449.046810:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574449.046813:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574449.046815:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574449.046817:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574449.046822:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046829:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574449.046830:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574449.046832:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574449.046834:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574449.046836:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574449.046838:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574449.046841:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574449.046843:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574449.046845:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574449.046847:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574449.046850:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574449.046852:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574449.046855:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574449.046857:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574449.046860:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574449.046864:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393135360, portal 10
00000100:00000001:1.0:1788574449.046865:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574449.046866:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:1.0:1788574449.046872:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574474 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046877:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574449.046879:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574449.046880:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574449.046882:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393135360, offset 0
00000400:00000200:1.0:1788574449.046885:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574449.046889:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574449.046892:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574449.046895:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574449.046898:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574449.046899:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574449.046901:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574449.046905:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330313:330313:256:4294967295] 192.168.202.110@tcp LPNI seq info [330270:330270:8:4294967295]
00000400:00000200:1.0:1788574449.046909:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574449.046916:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574449.046921:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574449.046925:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574449.046929:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574449.046932:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1788574449.046937:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574449.046940:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1788574449.046944:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574449.046947:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574449.046951:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574449.046955:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574449.046959:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574449.046961:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574449.046963:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574449.046966:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574474 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046972:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574449.046975:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574474 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.046980:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574449.046981:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574449.046983:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574449.046987:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574449.046990:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574449.046991:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306052647
00000100:00000001:1.0:1788574449.046994:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306053562
00000100:00000001:1.0:1788574449.046996:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306052647
00000100:00000001:1.0:1788574449.046998:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306053562
00000100:00000001:1.0:1788574449.046999:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574449.047000:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574449.047002:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574449.047004:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574449/real 1788574449]  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574449.047010:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574449.047011:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574449.047013:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574449.047015:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574449.047016:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574449.047018:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574449.047021:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574449.047027:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047032:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574449.047033:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047034:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574449.047035:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574449.047037:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047041:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574449.047043:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047047:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574449.047049:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047050:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574449.047050:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047052:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574449.047053:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574449.047055:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574449.047058:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574449.047059:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574449.047062:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574449.047067:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574449.047070:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574449.047072:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047082:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574449.047083:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574449.047085:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574449.047086:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574449.047088:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574449.047090:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135488:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574449.047093:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574449.047095:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574449.047097:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574449.047099:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574449.047103:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574449.047105:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574449.047108:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574449.047111:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574449.047115:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574449.047118:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393135488, portal 4
00000100:00000001:1.0:1788574449.047120:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574449.047121:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:1.0:1788574449.047129:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574474 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047136:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574449.047139:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574449.047142:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574449.047144:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393135488, offset 0
00000400:00000200:1.0:1788574449.047148:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574449.047152:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574449.047155:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574449.047157:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574449.047159:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574449.047161:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574449.047163:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574449.047166:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330314:330314:256:4294967295] 192.168.202.110@tcp LPNI seq info [330271:330271:8:4294967295]
00000400:00000200:1.0:1788574449.047169:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574449.047174:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574449.047179:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574449.047183:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574449.047187:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574449.047190:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1788574449.047195:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574449.047197:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1788574449.047201:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574449.047204:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574449.047207:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574449.047211:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574449.047214:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574449.047216:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574449.047219:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574449.047223:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574474 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047232:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574449.047236:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574474 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047244:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574449.047245:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574449.047249:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574449.047252:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574449.047256:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574449.047258:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306052793
00000100:00000001:1.0:1788574449.047261:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306053562
00000100:00000001:1.0:1788574449.047264:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306052793
00000100:00000001:1.0:1788574449.047266:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306053562
00000100:00000001:1.0:1788574449.047268:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574449.047271:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574449.047273:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574449.047277:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574449/real 1788574449]  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574449.047286:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574449.047288:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574449.047290:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574449.047293:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574449.047294:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574449.047297:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574449.047301:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574449.047309:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047317:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574449.047319:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047320:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574449.047322:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574449.047326:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047331:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574449.047333:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047338:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574449.047340:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047341:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574449.047342:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047343:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574449.047344:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574449.047346:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574449.047349:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574449.047350:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574449.047353:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574449.047358:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135488:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574449.047361:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047362:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574449.047364:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393135360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047368:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574449.047369:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574449.047371:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574449.047373:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574449.047374:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574449.047375:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574449.047376:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574449.047378:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:1.0:1788574449.047380:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574449.047381:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574449.047382:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:1.0:1788574449.047384:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574449.047385:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047386:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574449.047388:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393135488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574474 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574449.047424:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574449.047426:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574449.047428:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574449.047430:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574449.047432:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574449.047433:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574449.047435:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574449.047437:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:1.0:1788574449.047440:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574449.047441:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574449.047443:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:1.0:1788574449.047445:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574449.047447:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047449:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574449.047453:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574449.047455:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574449.047458:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574449.047460:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574450.005558:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306053562, at 4306053568
00000100:00000040:0.0:1788574450.005559:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306053562, at 4306053568
00000100:00000010:1.0:1788574450.005573:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9240.
00000100:00000010:0.0:1788574450.005579:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9b40.
00000100:00000001:1.0:1788574454.165707:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574454.165711:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574454.165713:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574454.165721:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.165730:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574454.165732:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574454.165736:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574454.165739:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574454.165742:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574454.165746:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393135616:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574454.165751:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574454.165755:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574454.165758:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574454.165761:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574454.165766:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574454.165768:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574454.165773:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574454.165777:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574454.165784:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574454.165791:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393135616, portal 25
00000100:00000001:1.0:1788574454.165793:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574454.165795:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:1.0:1788574454.165801:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574479 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.165810:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574454.165813:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574454.165816:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574454.165819:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393135616, offset 0
00000400:00000200:1.0:1788574454.165825:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574454.165833:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574454.165840:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574454.165845:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574454.165849:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574454.165853:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574454.165856:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574454.165861:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330315:330315:256:4294967295] 192.168.202.110@tcp LPNI seq info [330272:330272:8:4294967295]
00000400:00000200:1.0:1788574454.165866:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574454.165876:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574454.165883:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574454.165888:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574454.165894:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574454.165898:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1788574454.165905:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574454.165909:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1788574454.165914:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574454.165916:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574454.165922:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574454.165928:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574454.165933:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574454.165936:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574454.165940:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574454.165944:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574479 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.165954:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574454.165958:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574479 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.165965:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574454.165967:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574454.165973:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574454.165978:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574454.165980:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306058681
00000100:00000010:1.0:1788574454.165986:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9240.
00000100:00000001:1.0:1788574454.165992:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574454.165998:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306058681
00000100:00000010:1.0:1788574454.166001:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9f60.
00000100:00000001:1.0:1788574454.166004:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574454.166008:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574454.166011:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574454.166014:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574454.166019:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574454/real 1788574454]  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574454.166027:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574454.166028:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574454.166030:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574454.166032:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574454.166034:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574454.166035:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574454.166038:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574454.166043:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166048:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574454.166049:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166050:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574454.166052:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574454.166054:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166060:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574454.166063:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166068:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574454.166069:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166070:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574454.166071:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166073:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574454.166074:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574454.166076:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574454.166079:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574454.166080:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574454.166083:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574454.166089:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393135616:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574454.166092:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166093:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574454.166095:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393135616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166100:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574454.166102:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574454.166104:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574454.166106:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574454.166107:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574454.166108:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574454.166110:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574454.166112:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:1.0:1788574454.166115:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574454.166116:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574454.166118:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:1.0:1788574454.166128:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574454.166130:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166131:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166140:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574454.166142:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574454.166143:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574454.166144:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574454.166145:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574454.166148:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166153:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574454.166154:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574454.166155:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574454.166157:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574454.166158:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574454.166160:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135744:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574454.166162:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574454.166163:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574454.166165:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574454.166166:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574454.166168:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574454.166169:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574454.166172:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574454.166174:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574454.166176:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574454.166178:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393135744, portal 10
00000100:00000001:1.0:1788574454.166179:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574454.166180:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:1.0:1788574454.166185:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574479 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166189:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574454.166191:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574454.166193:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574454.166194:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393135744, offset 0
00000400:00000200:1.0:1788574454.166197:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574454.166200:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574454.166202:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574454.166205:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574454.166207:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574454.166208:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574454.166210:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574454.166212:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330316:330316:256:4294967295] 192.168.202.110@tcp LPNI seq info [330273:330273:8:4294967295]
00000400:00000200:1.0:1788574454.166215:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574454.166220:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574454.166223:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574454.166225:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574454.166227:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574454.166229:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1788574454.166233:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574454.166235:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1788574454.166237:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574454.166239:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574454.166241:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574454.166243:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574454.166245:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574454.166247:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574454.166248:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574454.166250:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574479 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166255:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574454.166257:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574479 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166262:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574454.166263:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574454.166265:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574454.166268:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574454.166270:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574454.166272:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306057766
00000100:00000001:1.0:1788574454.166274:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306058681
00000100:00000001:1.0:1788574454.166276:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306057766
00000100:00000001:1.0:1788574454.166277:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306058681
00000100:00000001:1.0:1788574454.166279:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574454.166280:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574454.166281:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574454.166283:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574454/real 1788574454]  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574454.166288:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574454.166289:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574454.166290:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574454.166291:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574454.166292:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574454.166293:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574454.166295:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574454.166301:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166305:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574454.166306:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166307:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574454.166308:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574454.166310:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166314:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574454.166316:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166321:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574454.166323:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166324:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574454.166325:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166326:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574454.166328:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574454.166329:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574454.166331:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574454.166332:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574454.166335:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574454.166339:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135744:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574454.166342:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574454.166343:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166348:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574454.166349:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574454.166350:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574454.166351:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574454.166352:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574454.166354:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135872:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574454.166356:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574454.166357:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574454.166358:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574454.166359:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574454.166361:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574454.166362:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574454.166364:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574454.166366:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574454.166368:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574454.166370:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393135872, portal 4
00000100:00000001:1.0:1788574454.166371:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574454.166372:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:1.0:1788574454.166376:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574479 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166379:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574454.166381:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574454.166382:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574454.166384:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393135872, offset 0
00000400:00000200:1.0:1788574454.166386:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574454.166388:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574454.166407:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574454.166410:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574454.166411:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574454.166413:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574454.166415:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574454.166417:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330317:330317:256:4294967295] 192.168.202.110@tcp LPNI seq info [330274:330274:8:4294967295]
00000400:00000200:1.0:1788574454.166420:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574454.166424:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574454.166427:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574454.166430:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574454.166432:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574454.166434:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1788574454.166437:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574454.166440:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1788574454.166442:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574454.166443:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574454.166446:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574454.166448:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574454.166450:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574454.166451:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574454.166453:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574454.166455:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574479 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166461:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574454.166463:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574479 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166467:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574454.166468:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574454.166469:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574454.166471:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574454.166473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574454.166474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306057913
00000100:00000001:1.0:1788574454.166476:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306058681
00000100:00000001:1.0:1788574454.166477:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306057913
00000100:00000001:1.0:1788574454.166479:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306058681
00000100:00000001:1.0:1788574454.166480:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574454.166482:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574454.166483:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574454.166485:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574454/real 1788574454]  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574454.166489:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574454.166490:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574454.166491:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574454.166492:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574454.166493:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574454.166495:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574454.166496:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574454.166502:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166510:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574454.166511:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166512:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574454.166513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574454.166515:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166519:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574454.166522:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166526:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574454.166527:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166528:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574454.166529:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166530:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574454.166531:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574454.166532:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574454.166534:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574454.166536:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574454.166538:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574454.166543:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393135872:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574454.166546:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166547:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574454.166549:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393135744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166553:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574454.166554:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574454.166555:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574454.166557:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574454.166558:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574454.166559:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574454.166561:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574454.166562:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:1.0:1788574454.166563:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574454.166564:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574454.166565:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:1.0:1788574454.166566:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574454.166567:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166568:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574454.166570:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393135872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574479 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574454.166575:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574454.166576:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574454.166577:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574454.166578:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574454.166579:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574454.166579:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574454.166580:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574454.166581:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:1.0:1788574454.166582:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574454.166583:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574454.166584:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:1.0:1788574454.166586:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574454.166586:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166587:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574454.166589:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574454.166590:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574454.166591:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574454.166593:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574455.125471:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306058681, at 4306058688
00000100:00000040:1.0:1788574455.125475:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306058681, at 4306058688
00000100:00000010:0.0:1788574455.125485:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9240.
00000100:00000010:1.0:1788574455.125488:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9f60.
00000100:00000001:1.0:1788574459.285693:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574459.285718:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574459.285723:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574459.285732:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.285751:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574459.285754:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574459.285757:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574459.285761:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574459.285766:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574459.285771:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393136000:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574459.285779:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574459.285783:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574459.285787:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574459.285790:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574459.285797:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574459.285800:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574459.285807:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574459.285812:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574459.285820:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574459.285829:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393136000, portal 25
00000100:00000001:1.0:1788574459.285834:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574459.285842:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:1.0:1788574459.285852:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.285871:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574459.285878:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574459.285883:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574459.285888:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393136000, offset 0
00000400:00000200:1.0:1788574459.285900:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574459.285912:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574459.285920:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574459.285928:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574459.285934:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574459.285942:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574459.285949:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574459.285955:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330318:330318:256:4294967295] 192.168.202.110@tcp LPNI seq info [330275:330275:8:4294967295]
00000400:00000200:1.0:1788574459.285965:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574459.285978:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574459.285987:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574459.285994:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574459.286004:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574459.286010:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1788574459.286020:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574459.286027:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1788574459.286034:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574459.286038:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574459.286045:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574459.286053:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574459.286059:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574459.286064:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574459.286069:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574459.286074:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286081:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574459.286083:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286088:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574459.286090:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574459.286094:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574459.286097:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574459.286099:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306063801
00000100:00000010:1.0:1788574459.286103:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9f60.
00000100:00000001:1.0:1788574459.286109:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574459.286120:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306063801
00000100:00000010:1.0:1788574459.286122:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f94e0.
00000100:00000001:1.0:1788574459.286124:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574459.286127:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574459.286129:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574459.286131:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574459.286134:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574459/real 1788574459]  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574459.286141:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574459.286142:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574459.286144:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574459.286147:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574459.286148:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574459.286151:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574459.286154:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574459.286160:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286168:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574459.286169:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286172:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574459.286174:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574459.286179:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286185:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574459.286190:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286197:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574459.286199:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286201:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574459.286202:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286204:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574459.286207:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574459.286209:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574459.286213:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574459.286215:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574459.286219:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574459.286228:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393136000:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574459.286233:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286236:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574459.286240:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393136000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286246:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574459.286248:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574459.286251:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574459.286254:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574459.286255:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574459.286257:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574459.286259:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574459.286261:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:1.0:1788574459.286265:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574459.286266:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574459.286268:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:1.0:1788574459.286271:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574459.286272:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286274:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286281:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574459.286282:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574459.286285:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574459.286286:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574459.286288:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574459.286291:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286299:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574459.286301:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574459.286303:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574459.286305:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574459.286307:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574459.286310:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574459.286313:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574459.286315:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574459.286317:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574459.286319:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574459.286322:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574459.286324:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574459.286328:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574459.286331:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574459.286335:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574459.286338:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393136128, portal 10
00000100:00000001:1.0:1788574459.286340:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574459.286341:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:1.0:1788574459.286346:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286350:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574459.286352:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574459.286354:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574459.286355:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393136128, offset 0
00000400:00000200:1.0:1788574459.286358:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574459.286361:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574459.286363:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574459.286365:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574459.286367:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574459.286369:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574459.286370:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574459.286373:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330319:330319:256:4294967295] 192.168.202.110@tcp LPNI seq info [330276:330276:8:4294967295]
00000400:00000200:1.0:1788574459.286375:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574459.286380:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574459.286383:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574459.286385:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574459.286388:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574459.286414:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1788574459.286418:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574459.286420:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1788574459.286422:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574459.286424:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574459.286427:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574459.286429:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574459.286431:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574459.286432:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574459.286434:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574459.286437:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286448:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574459.286450:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286454:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574459.286456:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574459.286458:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574459.286460:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574459.286463:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574459.286464:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306062887
00000100:00000001:1.0:1788574459.286467:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306063801
00000100:00000001:1.0:1788574459.286468:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306062887
00000100:00000001:1.0:1788574459.286469:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306063801
00000100:00000001:1.0:1788574459.286471:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574459.286473:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574459.286474:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574459.286476:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574459/real 1788574459]  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574459.286482:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574459.286483:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574459.286484:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574459.286486:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574459.286487:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574459.286488:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574459.286490:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574459.286495:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286499:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574459.286500:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286501:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574459.286502:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574459.286504:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286508:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574459.286511:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286515:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574459.286516:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286517:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574459.286518:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286519:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574459.286521:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574459.286522:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574459.286524:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574459.286526:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574459.286528:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574459.286534:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574459.286536:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574459.286538:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286542:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574459.286543:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574459.286544:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574459.286545:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574459.286547:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574459.286548:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574459.286550:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574459.286551:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574459.286552:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574459.286553:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574459.286556:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574459.286557:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574459.286559:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574459.286561:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574459.286563:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574459.286565:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393136256, portal 4
00000100:00000001:1.0:1788574459.286566:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574459.286567:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:1.0:1788574459.286571:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286575:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574459.286577:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574459.286578:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574459.286580:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393136256, offset 0
00000400:00000200:1.0:1788574459.286583:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574459.286585:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574459.286587:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574459.286589:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574459.286591:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574459.286592:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574459.286594:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574459.286596:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330320:330320:256:4294967295] 192.168.202.110@tcp LPNI seq info [330277:330277:8:4294967295]
00000400:00000200:1.0:1788574459.286599:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574459.286603:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574459.286606:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574459.286607:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574459.286610:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574459.286612:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1788574459.286614:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574459.286616:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1788574459.286619:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574459.286620:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574459.286622:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574459.286624:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574459.286626:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574459.286627:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574459.286628:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574459.286631:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286636:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574459.286638:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286642:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574459.286643:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574459.286644:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574459.286646:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574459.286648:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574459.286650:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306063033
00000100:00000001:1.0:1788574459.286652:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306063801
00000100:00000001:1.0:1788574459.286653:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306063033
00000100:00000001:1.0:1788574459.286654:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306063801
00000100:00000001:1.0:1788574459.286656:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574459.286657:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574459.286658:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574459.286660:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574459/real 1788574459]  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574459.286665:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574459.286665:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574459.286666:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574459.286668:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574459.286669:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574459.286670:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574459.286672:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574459.286678:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286681:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574459.286682:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286683:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574459.286684:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574459.286686:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286690:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574459.286692:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286709:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574459.286711:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286712:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574459.286713:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286714:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574459.286716:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574459.286718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574459.286719:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574459.286721:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574459.286724:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574459.286729:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574459.286731:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286732:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574459.286734:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393136128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286738:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574459.286739:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574459.286740:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574459.286742:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574459.286743:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574459.286744:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574459.286746:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574459.286747:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:1.0:1788574459.286749:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574459.286749:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574459.286750:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:1.0:1788574459.286752:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574459.286753:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286754:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574459.286756:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393136256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574459.286765:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574459.286765:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574459.286766:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574459.286768:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574459.286769:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574459.286769:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574459.286770:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574459.286771:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:1.0:1788574459.286772:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574459.286773:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574459.286774:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:1.0:1788574459.286776:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574459.286776:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286777:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574459.286780:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574459.286781:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574459.286782:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574459.286783:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574460.245494:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306063801, at 4306063808
00000100:00000040:0.0:1788574460.245495:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306063801, at 4306063808
00000100:00000010:1.0:1788574460.245509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f94e0.
00000100:00000010:0.0:1788574460.245512:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9f60.
00000100:00000001:1.0:1788574464.405919:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574464.405926:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574464.405929:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574464.405938:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.405952:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574464.405955:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574464.405961:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574464.405966:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574464.405970:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574464.405976:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393136384:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574464.405982:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574464.405988:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574464.405992:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574464.405999:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574464.406006:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574464.406009:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574464.406017:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574464.406023:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574464.406033:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574464.406043:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393136384, portal 25
00000100:00000001:1.0:1788574464.406049:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574464.406051:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000100:00000040:1.0:1788574464.406063:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574489 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406083:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574464.406089:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574464.406095:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574464.406102:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393136384, offset 0
00000400:00000200:1.0:1788574464.406115:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574464.406129:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574464.406139:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574464.406147:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574464.406154:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574464.406159:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574464.406164:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574464.406170:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330321:330321:256:4294967295] 192.168.202.110@tcp LPNI seq info [330278:330278:8:4294967295]
00000400:00000200:1.0:1788574464.406178:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574464.406193:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574464.406204:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574464.406212:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574464.406223:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574464.406231:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1788574464.406245:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574464.406253:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1788574464.406265:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574464.406271:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574464.406281:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574464.406291:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574464.406299:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574464.406302:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574464.406309:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574464.406315:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574489 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406330:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574464.406335:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574489 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406346:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574464.406350:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574464.406359:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574464.406367:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574464.406382:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306068921
00000100:00000010:1.0:1788574464.406435:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f94e0.
00000100:00000001:1.0:1788574464.406445:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574464.406458:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306068922
00000100:00000010:1.0:1788574464.406464:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9d80.
00000100:00000001:1.0:1788574464.406478:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574464.406486:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574464.406489:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574464.406492:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574464.406498:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574464/real 1788574464]  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574464.406508:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574464.406509:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574464.406511:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574464.406514:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574464.406515:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574464.406517:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574464.406520:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574464.406527:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406532:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574464.406533:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.406535:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574464.406537:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574464.406540:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406545:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574464.406548:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406554:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574464.406556:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.406557:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574464.406558:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.406560:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574464.406562:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574464.406564:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574464.406567:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574464.406568:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574464.406571:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574464.406576:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393136384:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574464.406581:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.406583:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574464.406589:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393136384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406600:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574464.406602:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574464.406605:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574464.406608:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574464.406609:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574464.406610:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574464.406612:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574464.406615:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:1.0:1788574464.406619:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574464.406621:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574464.406622:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:1.0:1788574464.406625:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574464.406626:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.406628:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.406635:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574464.406636:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574464.406638:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574464.406639:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574464.406640:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574464.406643:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406647:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574464.406649:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574464.406650:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574464.406652:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574464.406654:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574464.406656:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136512:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574464.406658:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574464.406660:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574464.406661:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574464.406662:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574464.406665:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574464.406666:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574464.406669:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574464.406671:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574464.406675:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574464.406679:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393136512, portal 10
00000100:00000001:1.0:1788574464.406681:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574464.406682:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:1.0:1788574464.406697:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574489 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406704:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574464.406707:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574464.406710:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574464.406713:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393136512, offset 0
00000400:00000200:1.0:1788574464.406718:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574464.406723:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574464.406727:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574464.406730:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574464.406731:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574464.406733:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574464.406735:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574464.406737:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330322:330322:256:4294967295] 192.168.202.110@tcp LPNI seq info [330279:330279:8:4294967295]
00000400:00000200:1.0:1788574464.406741:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574464.406745:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574464.406749:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574464.406751:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574464.406767:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574464.406769:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1788574464.406773:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574464.406775:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1788574464.406778:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574464.406779:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574464.406782:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574464.406784:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574464.406787:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574464.406788:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574464.406791:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574464.406796:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574489 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406805:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574464.406808:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574489 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406815:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574464.406818:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574464.406822:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574464.406826:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574464.406830:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574464.406833:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306068007
00000100:00000001:1.0:1788574464.406836:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306068921
00000100:00000001:1.0:1788574464.406839:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306068007
00000100:00000001:1.0:1788574464.406841:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306068922
00000100:00000001:1.0:1788574464.406843:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574464.406845:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574464.406847:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574464.406850:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574464/real 1788574464]  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574464.406856:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574464.406857:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574464.406858:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574464.406860:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574464.406862:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574464.406864:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574464.406866:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574464.406871:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406880:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574464.406881:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.406883:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574464.406884:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574464.406886:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406890:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574464.406892:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406897:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574464.406898:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.406899:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574464.406900:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.406901:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574464.406903:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574464.406905:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574464.406907:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574464.406909:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574464.406912:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574464.406917:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136512:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574464.406919:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574464.406921:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406926:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574464.406926:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574464.406928:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574464.406930:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574464.406931:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574464.406933:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136640:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574464.406936:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574464.406937:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574464.406938:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574464.406939:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574464.406942:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574464.406943:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574464.406946:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574464.406948:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574464.406951:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574464.406953:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393136640, portal 4
00000100:00000001:1.0:1788574464.406954:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574464.406955:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:1.0:1788574464.406960:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574489 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.406964:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574464.406966:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574464.406967:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574464.406969:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393136640, offset 0
00000400:00000200:1.0:1788574464.406972:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574464.406975:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574464.406978:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574464.406980:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574464.406982:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574464.406983:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574464.406985:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574464.406988:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330323:330323:256:4294967295] 192.168.202.110@tcp LPNI seq info [330280:330280:8:4294967295]
00000400:00000200:1.0:1788574464.406990:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574464.406995:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574464.406998:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574464.407001:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574464.407004:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574464.407006:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1788574464.407009:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574464.407011:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1788574464.407014:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574464.407015:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574464.407017:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574464.407019:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574464.407021:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574464.407023:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574464.407024:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574464.407026:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574489 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.407032:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574464.407034:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574489 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.407038:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574464.407040:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574464.407042:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574464.407044:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574464.407046:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574464.407048:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306068153
00000100:00000001:1.0:1788574464.407050:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306068921
00000100:00000001:1.0:1788574464.407053:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306068153
00000100:00000001:1.0:1788574464.407055:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306068922
00000100:00000001:1.0:1788574464.407057:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574464.407059:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574464.407061:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574464.407065:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574464/real 1788574464]  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574464.407075:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574464.407076:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574464.407078:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574464.407081:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574464.407083:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574464.407085:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574464.407089:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574464.407099:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.407105:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574464.407106:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.407107:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574464.407109:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574464.407111:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.407116:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574464.407118:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.407123:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574464.407125:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.407126:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574464.407127:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.407128:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574464.407129:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574464.407131:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574464.407133:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574464.407134:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574464.407137:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574464.407143:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136640:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574464.407145:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.407147:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574464.407149:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393136512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.407153:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574464.407154:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574464.407155:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574464.407157:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574464.407158:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574464.407159:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574464.407161:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574464.407162:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:1.0:1788574464.407164:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574464.407165:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574464.407166:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:1.0:1788574464.407168:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574464.407169:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.407170:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574464.407172:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393136640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574489 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574464.407177:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574464.407178:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574464.407179:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574464.407180:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574464.407181:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574464.407181:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574464.407182:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574464.407183:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:1.0:1788574464.407184:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574464.407186:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574464.407187:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:1.0:1788574464.407189:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574464.407190:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.407191:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574464.407194:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574464.407195:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574464.407196:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574464.407198:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574465.365529:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306068921, at 4306068928
00000100:00000040:1.0:1788574465.365537:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306068922, at 4306068928
00000100:00000010:0.0:1788574465.365548:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f94e0.
00000100:00000010:1.0:1788574465.365554:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9d80.
00000100:00080000:0.0:1788574469.525510:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574469.525518:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574469.525520:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574469.525524:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574469.525527:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574469.525533:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11400
00000400:00000200:0.0:1788574469.525540:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574469.525545:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574469.525548:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574469.525550:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525552:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574469.525554:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525556:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574469.525559:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574469.525560:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574469.525563:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574469.525565:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574469.525566:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574469.525568:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574469.525570:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525571:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574469.525572:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525573:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574469.525576:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574469.525578:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525580:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574469.525582:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574469.525585:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574469.525586:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574469.525588:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574469.525590:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574469.525597:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788574469.525599:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574469.525603:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574469.525604:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574469.525607:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574469.525609:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574469.525612:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574469.525618:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788574469.525623:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574469.525630:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574469.525637:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574469.525641:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574469.525645:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574469.525661:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574469.525663:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574469.525666:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574469.525667:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574469.525668:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574469.525670:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574469.525672:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11400
00000400:00000200:0.0:1788574469.525674:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574469.525676:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574469.525678:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574469.525680:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525682:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574469.525684:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525686:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574469.525687:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574469.525688:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574469.525689:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574469.525690:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:1.0:1788574469.525691:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788574469.525692:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574469.525697:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:1.0:1788574469.525697:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1788574469.525702:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574469.525702:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788574469.525705:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574469.525709:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525711:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:1.0:1788574469.525711:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788574469.525716:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574469.525721:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525725:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574469.525729:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1788574469.525730:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788574469.525734:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574469.525734:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574469.525740:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574469.525741:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788574469.525742:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1788574469.525745:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574469.525748:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574469.525751:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393136768:192.168.202.110@tcp:250:kworker.0
00000008:00100000:0.0:1788574469.525753:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1788574469.525755:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:0.0:1788574469.525758:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574469.525760:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1788574469.525761:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574469.525764:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574469.525766:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
02000000:00000001:1.0:1788574469.525767:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:0.0:1788574469.525769:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574469.525772:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574469.525773:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574469.525773:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
00000100:00000001:0.0:1788574469.525774:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574469.525777:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574469.525778:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574469.525780:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574469.525785:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1788574469.525786:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574469.525790:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1788574469.525791:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000100:00100000:0.0:1788574469.525796:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1788574469.525798:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000001:0.0:1788574469.525806:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1788574469.525807:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393136768, portal 25
00000100:00000001:0.0:1788574469.525808:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574469.525823:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574469.525825:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:1.0:1788574469.525831:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574494 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.525842:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574469.525843:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788574469.525848:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574469.525853:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574469.525858:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393136768, offset 0
00000400:00000200:1.0:1788574469.525867:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788574469.525868:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574469.525872:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574469.525874:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574469.525876:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574469.525877:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574469.525880:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574469.525885:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11400
00000400:00000200:1.0:1788574469.525885:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574469.525896:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574469.525900:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574469.525903:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574469.525906:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574469.525908:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574469.525911:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330324:330324:256:4294967295] 192.168.202.110@tcp LPNI seq info [330281:330281:8:4294967295]
00000400:00000200:1.0:1788574469.525915:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574469.525922:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1788574469.525931:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.0:1788574469.525931:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574469.525938:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000400:00000200:0.0:1788574469.525939:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574469.525944:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1788574469.525947:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788574469.525952:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:1.0:1788574469.525953:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:0.0:1788574469.525958:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1788574469.525961:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574469.525963:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:0.0:1788574469.525971:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000100:1.0:1788574469.525971:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574469.525974:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000100:00000001:0.0:1788574469.525976:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:1.0:1788574469.525980:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1788574469.525981:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574469.525985:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:1.0:1788574469.525986:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1788574469.525989:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574469.525992:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:1.0:1788574469.525992:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000020:00000001:0.0:1788574469.525995:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:1.0:1788574469.525996:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000020:00000001:0.0:1788574469.525998:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:1.0:1788574469.526001:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:0.0:1788574469.526003:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574469.526005:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1788574469.526008:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574494 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574469.526009:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574469.526013:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574469.526017:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574469.526021:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574469.526025:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574469.526029:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1788574469.526030:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1788574469.526032:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1788574469.526036:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574494 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788574469.526037:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574469.526041:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574469.526044:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574469.526050:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574469.526053:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574469.526060:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000100:00000001:1.0:1788574469.526061:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788574469.526064:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:1.0:1788574469.526065:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:0.0:1788574469.526069:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574469.526070:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574469.526071:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574469.526073:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574469.526075:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574469.526077:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574469.526078:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1788574469.526082:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:1.0:1788574469.526082:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306074041
00000100:00000001:0.0:1788574469.526087:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574469.526087:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9d80.
00000100:00100000:0.0:1788574469.526092:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574469.526095:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574469.526102:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574469.526103:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306074041
00000100:00000001:0.0:1788574469.526104:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574469.526109:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f95a0.
00000100:00000040:0.0:1788574469.526110:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526113:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574469.526118:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574469.526120:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574469.526122:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574469.526124:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1788574469.526125:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1788574469.526129:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00100000:1.0:1788574469.526132:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574469/real 1788574469]  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574469.526134:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11410)
00000400:00000001:1.0:1788574469.526144:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574469.526145:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574469.526147:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574469.526149:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574469.526151:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574469.526153:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574469.526156:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574469.526162:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526166:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574469.526168:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526169:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574469.526170:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574469.526173:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526178:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574469.526181:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526186:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574469.526188:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526189:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574469.526190:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526192:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574469.526193:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574469.526195:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574469.526197:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574469.526199:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574469.526202:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574469.526207:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393136768:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574469.526210:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526212:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574469.526216:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393136768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526223:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574469.526226:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574469.526229:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574469.526232:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574469.526234:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574469.526235:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574469.526238:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574469.526240:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5c00.
02000000:00000001:1.0:1788574469.526244:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574469.526246:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574469.526248:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:1.0:1788574469.526252:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574469.526253:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526255:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526263:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574469.526265:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574469.526268:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574469.526269:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574469.526270:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574469.526273:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526278:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574469.526279:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574469.526281:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574469.526283:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574469.526284:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574469.526286:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136896:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574469.526288:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574469.526290:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574469.526291:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574469.526292:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574469.526294:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574469.526296:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574469.526298:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574469.526301:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574469.526303:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574469.526306:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393136896, portal 10
00000100:00000001:1.0:1788574469.526307:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574469.526308:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523500544 : -130484186051072 : ffff895348710e00)
00000100:00000040:1.0:1788574469.526312:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574494 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526317:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574469.526319:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574469.526320:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574469.526322:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393136896, offset 0
00000400:00000200:1.0:1788574469.526325:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574469.526328:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574469.526330:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574469.526333:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574469.526335:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574469.526336:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574469.526338:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574469.526340:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330325:330325:256:4294967295] 192.168.202.110@tcp LPNI seq info [330282:330282:8:4294967295]
00000400:00000200:1.0:1788574469.526343:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574469.526348:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574469.526351:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574469.526353:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574469.526356:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574469.526358:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1788574469.526361:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574469.526362:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1788574469.526365:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574469.526366:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574469.526368:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574469.526371:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574469.526373:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574469.526375:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574469.526377:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574469.526381:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574494 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526436:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574469.526448:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574494 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526454:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574469.526455:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574469.526459:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574469.526462:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574469.526465:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574469.526466:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306073127
00000100:00000001:1.0:1788574469.526468:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306074041
00000100:00000001:1.0:1788574469.526470:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306073127
00000100:00000001:1.0:1788574469.526472:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306074041
00000100:00000001:1.0:1788574469.526473:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574469.526475:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574469.526476:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574469.526479:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574469/real 1788574469]  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574469.526485:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574469.526486:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574469.526487:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574469.526489:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574469.526490:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574469.526491:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574469.526493:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574469.526499:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526503:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574469.526504:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526505:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574469.526506:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574469.526509:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526513:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574469.526516:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526521:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574469.526524:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526525:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574469.526526:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526527:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574469.526529:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574469.526532:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574469.526535:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574469.526537:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574469.526541:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574469.526550:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393136896:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574469.526554:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574469.526558:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526566:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574469.526567:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574469.526569:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574469.526571:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574469.526573:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574469.526576:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574469.526580:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574469.526582:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574469.526583:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574469.526585:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574469.526587:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574469.526589:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574469.526592:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574469.526594:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574469.526596:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574469.526598:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393137024, portal 4
00000100:00000001:1.0:1788574469.526599:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574469.526600:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523505920 : -130484186045696 : ffff895348712300)
00000100:00000040:1.0:1788574469.526605:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574494 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526609:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574469.526611:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574469.526612:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574469.526614:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393137024, offset 0
00000400:00000200:1.0:1788574469.526617:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574469.526619:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574469.526622:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574469.526624:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574469.526626:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574469.526627:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574469.526629:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574469.526631:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330326:330326:256:4294967295] 192.168.202.110@tcp LPNI seq info [330283:330283:8:4294967295]
00000400:00000200:1.0:1788574469.526634:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574469.526638:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574469.526641:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574469.526643:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574469.526646:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574469.526647:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1788574469.526650:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574469.526652:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1788574469.526654:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574469.526656:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574469.526658:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574469.526660:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574469.526662:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574469.526664:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574469.526666:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574469.526668:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574494 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526673:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574469.526675:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574494 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526679:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574469.526681:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574469.526683:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574469.526685:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574469.526687:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574469.526688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306073273
00000100:00000001:1.0:1788574469.526690:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306074041
00000100:00000001:1.0:1788574469.526691:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306073273
00000100:00000001:1.0:1788574469.526693:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306074041
00000100:00000001:1.0:1788574469.526694:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574469.526695:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574469.526696:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574469.526699:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574469/real 1788574469]  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574469.526704:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574469.526705:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574469.526706:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574469.526708:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574469.526709:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574469.526710:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574469.526712:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574469.526717:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526726:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574469.526727:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526728:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574469.526729:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574469.526731:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526735:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574469.526738:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526742:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574469.526743:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526744:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574469.526745:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526746:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574469.526747:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574469.526749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574469.526751:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574469.526752:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574469.526755:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574469.526760:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574469.526762:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526764:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574469.526766:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710e00 x1875444393136896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526769:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574469.526771:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574469.526772:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574469.526774:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574469.526775:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574469.526776:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574469.526777:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574469.526779:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:1.0:1788574469.526780:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574469.526781:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574469.526782:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710e00.
00000100:00000001:1.0:1788574469.526784:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574469.526785:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526786:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574469.526788:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712300 x1875444393137024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574494 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574469.526792:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574469.526793:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574469.526794:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574469.526795:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574469.526796:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574469.526797:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574469.526798:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574469.526799:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:1.0:1788574469.526800:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574469.526801:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574469.526801:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712300.
00000100:00000001:1.0:1788574469.526803:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574469.526804:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526805:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574469.526807:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574469.526808:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574469.526825:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574469.526827:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574470.485510:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306074041, at 4306074048
00000100:00000040:1.0:1788574470.485510:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306074041, at 4306074048
00000100:00000010:0.0:1788574470.485526:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9d80.
00000100:00000010:1.0:1788574470.485531:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f95a0.
00000100:00080000:0.0:1788574474.645500:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574474.645512:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574474.645515:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574474.645519:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574474.645523:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574474.645530:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11405
00000400:00000200:0.0:1788574474.645540:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574474.645546:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574474.645549:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574474.645552:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645555:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574474.645556:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645558:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574474.645561:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574474.645563:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574474.645565:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574474.645567:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574474.645569:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574474.645570:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574474.645572:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645574:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574474.645575:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645576:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574474.645579:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574474.645581:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645582:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574474.645585:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574474.645593:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574474.645595:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574474.645597:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574474.645599:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574474.645606:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788574474.645609:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574474.645612:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574474.645613:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574474.645615:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574474.645618:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574474.645622:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574474.645627:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574474.645632:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574474.645639:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574474.645652:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574474.645655:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574474.645658:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574474.645675:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574474.645679:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574474.645682:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574474.645684:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574474.645686:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574474.645689:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574474.645692:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11405
00000400:00000200:0.0:1788574474.645696:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574474.645699:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574474.645702:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574474.645705:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645707:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574474.645709:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645712:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574474.645714:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574474.645716:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574474.645717:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1788574474.645718:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1788574474.645720:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574474.645722:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:1.0:1788574474.645722:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788574474.645724:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:1.0:1788574474.645725:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574474.645728:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645730:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574474.645731:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574474.645732:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645734:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574474.645737:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574474.645740:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574474.645740:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574474.645742:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574474.645743:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574474.645744:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574474.645745:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574474.645748:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574474.645749:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574474.645751:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574474.645752:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574474.645754:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:1.0:1788574474.645754:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393137152:192.168.202.110@tcp:250:kworker.0
00000008:00100000:0.0:1788574474.645758:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1788574474.645759:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:0.0:1788574474.645761:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574474.645763:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574474.645763:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574474.645765:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574474.645766:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
02000000:00000001:1.0:1788574474.645767:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:0.0:1788574474.645769:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574474.645771:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574474.645773:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574474.645773:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
00000100:00000001:0.0:1788574474.645775:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574474.645775:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574474.645777:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574474.645778:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574474.645780:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1788574474.645782:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000400:00000010:1.0:1788574474.645784:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000100:00000001:0.0:1788574474.645785:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574474.645790:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1788574474.645790:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574474.645796:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393137152, portal 25
00000100:00000001:0.0:1788574474.645797:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574474.645799:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.645799:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574474.645801:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00000040:0.0:1788574474.645803:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788574474.645806:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.645815:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574474.645818:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645819:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1788574474.645821:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00080000:0.0:1788574474.645822:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:1.0:1788574474.645824:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393137152, offset 0
00000100:00080000:0.0:1788574474.645825:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574474.645827:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574474.645828:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574474.645831:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:1.0:1788574474.645831:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788574474.645833:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11405
00000400:00000200:0.0:1788574474.645837:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574474.645839:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574474.645842:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1788574474.645842:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788574474.645845:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645847:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1788574474.645848:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:0.0:1788574474.645849:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645851:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574474.645853:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574474.645854:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:1.0:1788574474.645854:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:0.0:1788574474.645856:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574474.645858:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:1.0:1788574474.645858:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:0.0:1788574474.645860:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574474.645862:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:1.0:1788574474.645862:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1788574474.645865:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:1.0:1788574474.645865:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574474.645889:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330327:330327:256:4294967295] 192.168.202.110@tcp LPNI seq info [330284:330284:8:4294967295]
00000400:00000200:1.0:1788574474.645894:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574474.645896:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574474.645898:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645901:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574474.645903:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:1.0:1788574474.645904:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1788574474.645906:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645908:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574474.645911:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:1.0:1788574474.645911:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788574474.645914:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574474.645916:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1788574474.645917:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000100:00000001:0.0:1788574474.645918:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574474.645920:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574474.645923:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574474.645924:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574474.645925:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574474.645928:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000800:00000200:1.0:1788574474.645928:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000040:0.0:1788574474.645931:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574474.645934:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:1.1:1788574474.645934:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1788574474.645935:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574474.645937:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574474.645938:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:1.1:1788574474.645938:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
02000000:00000001:0.0:1788574474.645940:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574474.645944:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000800:00000100:1.0:1788574474.645947:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788574474.645948:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1788574474.645950:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000100:00100000:0.0:1788574474.645952:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1788574474.645957:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1788574474.645959:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574474.645962:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574474.645962:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1788574474.645965:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574474.645968:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574474.645970:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:0.0:1788574474.645973:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.645975:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:0.0:1788574474.645976:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:1.0:1788574474.645979:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574474.645980:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574474.645985:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11415)
00000100:00000001:1.0:1788574474.645989:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574474.645992:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574499 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646000:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.646002:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574474.646008:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574474.646012:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574474.646015:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306079161
00000100:00000010:1.0:1788574474.646021:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f95a0.
00000100:00000001:1.0:1788574474.646028:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574474.646046:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306079161
00000100:00000010:1.0:1788574474.646050:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f96c0.
00000100:00000001:1.0:1788574474.646054:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574474.646058:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574474.646061:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.646064:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574474.646074:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574474/real 1788574474]  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574474.646085:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574474.646087:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574474.646089:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574474.646092:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574474.646094:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574474.646097:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574474.646099:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574474.646105:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646110:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574474.646111:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646112:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574474.646114:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574474.646117:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646121:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574474.646124:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646129:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574474.646131:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646132:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574474.646133:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646135:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574474.646136:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574474.646138:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574474.646141:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574474.646143:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574474.646145:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574474.646151:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393137152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574474.646154:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646155:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574474.646157:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393137152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646161:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574474.646163:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574474.646165:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574474.646167:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574474.646168:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574474.646169:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574474.646171:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574474.646173:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:1.0:1788574474.646175:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574474.646177:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574474.646178:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:1.0:1788574474.646181:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574474.646182:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646183:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646189:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574474.646191:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.646192:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574474.646193:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574474.646194:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574474.646197:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646201:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574474.646202:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574474.646204:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574474.646205:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574474.646207:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574474.646209:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574474.646211:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574474.646213:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574474.646214:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574474.646215:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574474.646218:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574474.646220:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574474.646224:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574474.646227:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574474.646231:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574474.646234:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393137280, portal 10
00000100:00000001:1.0:1788574474.646236:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574474.646237:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00000040:1.0:1788574474.646244:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646252:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574474.646254:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574474.646257:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574474.646260:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393137280, offset 0
00000400:00000200:1.0:1788574474.646264:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574474.646269:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574474.646273:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574474.646276:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574474.646279:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574474.646281:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574474.646282:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574474.646284:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330328:330328:256:4294967295] 192.168.202.110@tcp LPNI seq info [330285:330285:8:4294967295]
00000400:00000200:1.0:1788574474.646287:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574474.646292:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574474.646296:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574474.646298:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574474.646301:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574474.646303:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1788574474.646306:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574474.646308:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1788574474.646311:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574474.646313:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574474.646315:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574474.646317:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574474.646320:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574474.646321:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574474.646323:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574474.646325:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646331:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574474.646333:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574499 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646337:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.646338:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574474.646341:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574474.646343:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574474.646346:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574474.646347:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306078246
00000100:00000001:1.0:1788574474.646349:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306079161
00000100:00000001:1.0:1788574474.646351:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306078246
00000100:00000001:1.0:1788574474.646352:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306079161
00000100:00000001:1.0:1788574474.646353:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574474.646355:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.646356:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574474.646358:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574474/real 1788574474]  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574474.646364:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574474.646365:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574474.646366:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574474.646368:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574474.646369:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574474.646370:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574474.646372:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574474.646378:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646382:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574474.646383:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646384:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574474.646384:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574474.646387:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646424:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574474.646429:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646437:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574474.646439:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646441:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574474.646442:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646444:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574474.646447:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574474.646449:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574474.646452:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574474.646454:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574474.646457:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574474.646462:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574474.646465:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574474.646467:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646471:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574474.646472:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574474.646474:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574474.646475:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574474.646476:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574474.646478:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137408:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574474.646480:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574474.646482:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574474.646483:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574474.646484:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574474.646486:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574474.646488:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574474.646491:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574474.646493:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574474.646495:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574474.646497:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393137408, portal 4
00000100:00000001:1.0:1788574474.646499:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574474.646500:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:1.0:1788574474.646504:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646508:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574474.646510:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574474.646511:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574474.646513:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393137408, offset 0
00000400:00000200:1.0:1788574474.646516:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574474.646519:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574474.646521:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574474.646523:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574474.646525:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574474.646527:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574474.646529:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574474.646531:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330329:330329:256:4294967295] 192.168.202.110@tcp LPNI seq info [330286:330286:8:4294967295]
00000400:00000200:1.0:1788574474.646533:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574474.646538:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574474.646541:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574474.646543:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574474.646545:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574474.646547:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1788574474.646550:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574474.646553:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1788574474.646555:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574474.646557:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574474.646559:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574474.646561:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574474.646564:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574474.646565:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574474.646566:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574474.646568:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646573:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574474.646575:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574499 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646580:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.646581:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574474.646583:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574474.646585:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574474.646587:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574474.646588:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306078393
00000100:00000001:1.0:1788574474.646590:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306079161
00000100:00000001:1.0:1788574474.646592:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306078393
00000100:00000001:1.0:1788574474.646593:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306079161
00000100:00000001:1.0:1788574474.646594:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574474.646596:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.646597:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574474.646599:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574474/real 1788574474]  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574474.646603:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574474.646604:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574474.646606:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574474.646607:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574474.646608:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574474.646609:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574474.646611:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574474.646616:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646620:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574474.646621:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646622:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574474.646623:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574474.646625:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646629:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574474.646631:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646635:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574474.646637:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646638:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574474.646639:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646640:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574474.646641:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574474.646643:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574474.646645:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574474.646646:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574474.646649:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574474.646657:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137408:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574474.646661:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646663:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574474.646666:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393137280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646673:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574474.646674:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574474.646676:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574474.646679:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574474.646680:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574474.646682:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574474.646684:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574474.646686:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:1.0:1788574474.646689:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574474.646690:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574474.646692:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:1.0:1788574474.646694:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574474.646696:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646697:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574474.646701:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393137408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574499 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574474.646711:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574474.646712:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574474.646713:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574474.646715:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574474.646716:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574474.646716:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574474.646718:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574474.646718:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:1.0:1788574474.646720:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574474.646720:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574474.646721:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:1.0:1788574474.646723:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574474.646723:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646725:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574474.646727:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574474.646728:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574474.646729:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574474.646731:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574475.605504:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306079161, at 4306079168
00000100:00000040:1.0:1788574475.605504:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306079161, at 4306079168
00000100:00000010:0.0:1788574475.605519:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f95a0.
00000100:00000010:1.0:1788574475.605522:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f96c0.
00000100:00080000:0.0:1788574479.765459:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574479.765468:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574479.765470:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574479.765475:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574479.765479:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574479.765485:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11410
00000400:00000200:0.0:1788574479.765493:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574479.765499:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574479.765503:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574479.765506:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.765509:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574479.765513:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.765515:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574479.765518:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574479.765520:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574479.765522:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574479.765525:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574479.765534:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574479.765536:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574479.765538:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.765540:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574479.765541:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.765542:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574479.765545:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574479.765547:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574479.765549:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574479.765551:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574479.765553:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574479.765555:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574479.765557:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574479.765559:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574479.765566:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574479.765569:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574479.765572:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574479.765574:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574479.765576:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574479.765578:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574479.765581:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574479.765587:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788574479.765593:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574479.765600:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574479.765608:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574479.765611:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574479.765615:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574479.765633:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574479.765636:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574479.765638:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574479.765639:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574479.765640:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574479.765642:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574479.765644:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11410
00000400:00000200:0.0:1788574479.765649:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574479.765652:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574479.765654:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574479.765656:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.765658:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574479.765660:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.765662:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:1.0:1788574479.765665:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788574479.765666:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574479.765668:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574479.765673:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574479.765675:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:1.0:1788574479.765675:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788574479.765681:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:1.0:1788574479.765681:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1788574479.765686:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574479.765694:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:1.0:1788574479.765697:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574479.765700:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574479.765711:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574479.765719:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574479.765721:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:1.0:1788574479.765723:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574479.765728:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1788574479.765730:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:1.0:1788574479.765732:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574479.765737:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574479.765741:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:1.0:1788574479.765745:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393137536:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574479.765749:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574479.765752:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574479.765756:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574479.765759:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574479.765763:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574479.765765:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574479.765766:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1788574479.765772:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574479.765776:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
00000100:00000001:0.0:1788574479.765777:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1788574479.765782:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1788574479.765785:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574479.765789:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574479.765791:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574479.765794:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1788574479.765798:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000400:00000010:1.0:1788574479.765799:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000020:00000040:0.0:1788574479.765805:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574479.765808:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574479.765810:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1788574479.765811:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000001:0.0:1788574479.765812:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574479.765815:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574479.765817:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1788574479.765820:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393137536, portal 25
02000000:00000010:0.0:1788574479.765822:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:1.0:1788574479.765825:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574479.765828:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000001:0.0:1788574479.765829:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574479.765853:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574479.765854:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574479.765884:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574479.765891:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788574479.765892:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1788574479.765896:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000001:0.0:1788574479.765898:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1788574479.765900:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393137536, offset 0
00000400:00000200:1.0:1788574479.765911:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788574479.765912:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574479.765922:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574479.765947:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574479.765948:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:0.0:1788574479.765957:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1788574479.765961:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1788574479.765963:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574479.765966:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1788574479.765968:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1788574479.765971:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1788574479.765974:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574479.765980:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.765981:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:1.0:1788574479.765986:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330330:330330:256:4294967295] 192.168.202.110@tcp LPNI seq info [330287:330287:8:4294967295]
00000100:00080000:0.0:1788574479.765993:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11410
00000400:00000200:1.0:1788574479.765996:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574479.766010:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1788574479.766023:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.0:1788574479.766025:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574479.766031:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:1.0:1788574479.766033:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000100:00080000:0.0:1788574479.766034:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574479.766041:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.766049:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1788574479.766050:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788574479.766055:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:1.0:1788574479.766056:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1788574479.766059:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574479.766065:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574479.766066:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574479.766071:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:1.1:1788574479.766071:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1788574479.766081:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:1.1:1788574479.766081:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:0.0:1788574479.766091:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000100:1.0:1788574479.766096:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788574479.766098:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000010:1.0:1788574479.766101:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000100:00000040:0.0:1788574479.766106:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.766109:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1788574479.766111:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1788574479.766115:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574479.766119:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:1.0:1788574479.766126:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:0.0:1788574479.766128:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574479.766139:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1788574479.766139:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574479.766146:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:0.0:1788574479.766147:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574479.766153:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1788574479.766156:0:2361:0:(events.c:35:request_out_callback()) Process entered
02000000:00000001:0.0:1788574479.766157:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574479.766160:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1788574479.766164:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574479.766165:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574479.766170:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574479.766179:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574479.766187:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574479.766196:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000100:00000001:1.0:1788574479.766200:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1788574479.766202:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000040:1.0:1788574479.766206:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788574479.766210:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574479.766217:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574479.766220:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574479.766223:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1788574479.766223:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:0.0:1788574479.766234:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1788574479.766237:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574479.766240:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574479.766246:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1788574479.766252:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:1.0:1788574479.766252:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306084281
00000100:00000001:0.0:1788574479.766263:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574479.766264:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f96c0.
00000100:00100000:0.0:1788574479.766272:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574479.766275:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574479.766284:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306084281
00000100:00000001:0.0:1788574479.766286:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000010:1.0:1788574479.766288:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9900.
00000100:00000001:1.0:1788574479.766292:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574479.766293:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574479.766297:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574479.766300:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574479.766304:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766304:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574479.766311:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574479/real 1788574479]  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574479.766364:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1788574479.766365:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574479.766369:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1788574479.766373:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1788574479.766373:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574479.766379:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574479.766385:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000040:0.0:1788574479.766386:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:1.0:1788574479.766389:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574479.766421:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574479.766434:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11421)
00000100:00000200:1.0:1788574479.766435:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766448:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574479.766451:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.766454:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574479.766458:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574479.766465:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766474:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574479.766487:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766495:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574479.766499:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.766501:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574479.766502:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.766506:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574479.766508:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574479.766512:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574479.766517:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574479.766520:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574479.766525:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574479.766536:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393137536:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574479.766553:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.766556:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574479.766560:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393137536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766575:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574479.766578:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574479.766582:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574479.766586:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574479.766589:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574479.766592:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574479.766594:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574479.766598:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:1.0:1788574479.766602:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574479.766605:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574479.766608:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:1.0:1788574479.766612:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574479.766613:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.766616:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.766626:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574479.766629:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574479.766632:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574479.766635:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574479.766637:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574479.766642:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766651:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574479.766653:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574479.766655:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574479.766658:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574479.766661:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574479.766664:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137664:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574479.766668:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574479.766671:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574479.766672:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574479.766674:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574479.766677:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574479.766679:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574479.766684:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574479.766688:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574479.766692:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574479.766696:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393137664, portal 10
00000100:00000001:1.0:1788574479.766698:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574479.766699:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
00000100:00000040:1.0:1788574479.766708:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766716:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574479.766718:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574479.766721:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574479.766724:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393137664, offset 0
00000400:00000200:1.0:1788574479.766728:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574479.766734:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574479.766739:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574479.766742:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574479.766745:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574479.766748:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574479.766751:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574479.766754:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330331:330331:256:4294967295] 192.168.202.110@tcp LPNI seq info [330288:330288:8:4294967295]
00000400:00000200:1.0:1788574479.766759:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574479.766767:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574479.766772:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574479.766776:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574479.766780:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574479.766783:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1788574479.766789:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574479.766792:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1788574479.766797:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574479.766799:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574479.766803:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574479.766807:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574479.766810:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574479.766813:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574479.766816:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574479.766820:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766829:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574479.766832:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766840:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574479.766842:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574479.766846:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574479.766850:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574479.766854:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574479.766856:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306083367
00000100:00000001:1.0:1788574479.766860:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306084281
00000100:00000001:1.0:1788574479.766862:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306083367
00000100:00000001:1.0:1788574479.766865:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306084281
00000100:00000001:1.0:1788574479.766867:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574479.766870:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574479.766872:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574479.766877:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574479/real 1788574479]  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574479.766885:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574479.766887:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574479.766889:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574479.766892:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574479.766894:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574479.766897:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574479.766901:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574479.766910:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766940:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574479.766942:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.766944:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574479.766946:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574479.766950:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766959:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574479.766964:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.766971:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574479.766974:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.766976:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574479.766977:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.766980:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574479.766982:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574479.766985:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574479.766989:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574479.766992:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574479.766997:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574479.767005:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137664:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574479.767010:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574479.767014:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.767022:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574479.767023:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574479.767027:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574479.767029:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574479.767031:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574479.767034:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137792:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574479.767038:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574479.767041:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574479.767043:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574479.767045:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574479.767049:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574479.767051:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574479.767056:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574479.767059:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574479.767063:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574479.767066:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393137792, portal 4
00000100:00000001:1.0:1788574479.767069:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574479.767070:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000040:1.0:1788574479.767078:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.767085:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574479.767088:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574479.767091:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574479.767094:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393137792, offset 0
00000400:00000200:1.0:1788574479.767099:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574479.767103:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574479.767107:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574479.767110:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574479.767113:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574479.767116:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574479.767119:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574479.767122:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330332:330332:256:4294967295] 192.168.202.110@tcp LPNI seq info [330289:330289:8:4294967295]
00000400:00000200:1.0:1788574479.767127:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574479.767135:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574479.767140:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574479.767144:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574479.767148:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574479.767151:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1788574479.767156:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574479.767159:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1788574479.767164:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574479.767166:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574479.767169:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574479.767173:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574479.767176:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574479.767178:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574479.767181:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574479.767185:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.767193:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574479.767197:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.767204:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574479.767206:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574479.767209:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574479.767213:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574479.767216:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574479.767219:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306083513
00000100:00000001:1.0:1788574479.767222:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306084281
00000100:00000001:1.0:1788574479.767224:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306083513
00000100:00000001:1.0:1788574479.767227:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306084281
00000100:00000001:1.0:1788574479.767229:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574479.767231:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574479.767233:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574479.767237:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574479/real 1788574479]  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574479.767246:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574479.767247:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574479.767249:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574479.767252:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574479.767254:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574479.767257:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574479.767260:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574479.767270:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.767277:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574479.767279:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.767281:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574479.767282:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574479.767286:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.767294:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574479.767298:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.767306:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574479.767308:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.767310:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574479.767312:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.767314:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574479.767316:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574479.767319:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574479.767323:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574479.767325:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574479.767331:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574479.767339:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393137792:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574479.767343:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.767345:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574479.767348:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393137664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.767356:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574479.767357:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574479.767360:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574479.767363:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574479.767365:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574479.767366:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574479.767369:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574479.767370:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:1.0:1788574479.767374:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574479.767375:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574479.767377:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:1.0:1788574479.767380:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574479.767381:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.767383:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574479.767387:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393137792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574479.767420:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574479.767421:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574479.767423:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574479.767427:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574479.767429:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574479.767430:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574479.767432:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574479.767434:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:1.0:1788574479.767436:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574479.767438:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574479.767440:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:1.0:1788574479.767443:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574479.767444:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.767446:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574479.767450:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574479.767452:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574479.767454:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574479.767456:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574480.725482:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306084281, at 4306084288
00000100:00000010:1.0:1788574480.725512:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9900.
00000100:00000040:0.0:1788574480.725513:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306084281, at 4306084288
00000100:00000010:0.0:1788574480.725522:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f96c0.
00000800:00000200:1.0:1788574481.813520:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574481.813526:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574481.813530:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574481.813701:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574481.813717:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574481.813718:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574481.813732:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574481.813736:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.885719:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574484.885725:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574484.885730:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574484.885740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.885763:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574484.885768:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574484.885775:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574484.885780:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574484.885785:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574484.885794:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393137920:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574484.885803:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574484.885809:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574484.885814:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574484.885819:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574484.885828:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574484.885833:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574484.885845:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574484.885853:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574484.885863:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574484.885875:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393137920, portal 25
00000100:00000001:1.0:1788574484.885881:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574484.885885:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000040:1.0:1788574484.885898:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.885919:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574484.885940:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574484.885946:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574484.885954:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393137920, offset 0
00000400:00000200:1.0:1788574484.885968:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574484.885992:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574484.886003:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574484.886010:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574484.886015:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574484.886020:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574484.886029:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574484.886036:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330333:330333:256:4294967295] 192.168.202.110@tcp LPNI seq info [330290:330290:8:4294967295]
00000400:00000200:1.0:1788574484.886044:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574484.886060:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574484.886072:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574484.886078:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574484.886087:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574484.886091:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1788574484.886096:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574484.886099:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1788574484.886102:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574484.886104:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574484.886109:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574484.886112:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574484.886117:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574484.886119:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574484.886123:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574484.886126:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886134:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574484.886136:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886141:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574484.886143:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574484.886147:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574484.886150:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574484.886153:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306089401
00000100:00000010:1.0:1788574484.886157:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9900.
00000100:00000001:1.0:1788574484.886162:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574484.886172:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306089401
00000100:00000010:1.0:1788574484.886174:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9060.
00000100:00000001:1.0:1788574484.886177:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574484.886179:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574484.886181:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574484.886183:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574484.886186:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574484/real 1788574484]  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574484.886193:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574484.886195:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574484.886197:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574484.886199:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574484.886200:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574484.886202:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574484.886205:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574484.886212:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886217:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574484.886218:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886220:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574484.886221:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574484.886224:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886229:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574484.886232:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886236:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574484.886238:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886239:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574484.886240:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886241:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574484.886243:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574484.886245:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574484.886248:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574484.886250:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574484.886252:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574484.886257:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393137920:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574484.886260:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886261:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574484.886263:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393137920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886267:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574484.886269:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574484.886272:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574484.886274:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574484.886275:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574484.886276:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574484.886278:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574484.886280:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:1.0:1788574484.886282:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574484.886284:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574484.886285:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:1.0:1788574484.886287:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574484.886288:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886290:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886296:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574484.886297:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574484.886298:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574484.886300:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574484.886301:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574484.886303:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886308:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574484.886309:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574484.886310:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574484.886312:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574484.886313:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574484.886315:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574484.886317:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574484.886318:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574484.886319:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574484.886320:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574484.886322:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574484.886323:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574484.886326:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574484.886328:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574484.886330:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574484.886332:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393138048, portal 10
00000100:00000001:1.0:1788574484.886334:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574484.886334:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:1.0:1788574484.886339:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886344:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574484.886345:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574484.886347:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574484.886348:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393138048, offset 0
00000400:00000200:1.0:1788574484.886351:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574484.886353:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574484.886356:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574484.886358:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574484.886360:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574484.886361:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574484.886363:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574484.886365:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330334:330334:256:4294967295] 192.168.202.110@tcp LPNI seq info [330291:330291:8:4294967295]
00000400:00000200:1.0:1788574484.886368:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574484.886372:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574484.886375:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574484.886377:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574484.886380:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574484.886382:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1788574484.886384:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574484.886386:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1788574484.886388:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574484.886390:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574484.886409:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574484.886411:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574484.886414:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574484.886415:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574484.886416:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574484.886419:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886429:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574484.886431:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886435:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574484.886436:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574484.886438:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574484.886440:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574484.886442:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574484.886444:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306088487
00000100:00000001:1.0:1788574484.886445:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306089401
00000100:00000001:1.0:1788574484.886447:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306088487
00000100:00000001:1.0:1788574484.886449:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306089401
00000100:00000001:1.0:1788574484.886450:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574484.886452:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574484.886453:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574484.886455:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574484/real 1788574484]  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574484.886460:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574484.886461:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574484.886462:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574484.886463:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574484.886464:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574484.886466:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574484.886468:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574484.886473:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886476:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574484.886478:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886479:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574484.886480:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574484.886482:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886486:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574484.886489:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886493:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574484.886494:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886495:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574484.886496:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886497:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574484.886498:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574484.886500:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574484.886502:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574484.886503:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574484.886505:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574484.886510:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574484.886512:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574484.886515:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886519:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574484.886520:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574484.886521:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574484.886523:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574484.886524:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574484.886526:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574484.886528:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574484.886529:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574484.886530:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574484.886532:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574484.886534:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574484.886535:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574484.886537:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574484.886539:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574484.886541:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574484.886543:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393138176, portal 4
00000100:00000001:1.0:1788574484.886544:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574484.886545:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:1.0:1788574484.886549:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886553:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574484.886555:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574484.886556:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574484.886557:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393138176, offset 0
00000400:00000200:1.0:1788574484.886568:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574484.886570:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574484.886573:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574484.886574:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574484.886576:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574484.886578:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574484.886579:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574484.886581:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330335:330335:256:4294967295] 192.168.202.110@tcp LPNI seq info [330292:330292:8:4294967295]
00000400:00000200:1.0:1788574484.886584:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574484.886588:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574484.886591:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574484.886593:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574484.886596:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574484.886598:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1788574484.886601:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574484.886602:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1788574484.886605:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574484.886606:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574484.886608:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574484.886610:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574484.886612:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574484.886613:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574484.886615:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574484.886617:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886622:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574484.886624:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886628:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574484.886629:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574484.886631:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574484.886633:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574484.886635:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574484.886636:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306088633
00000100:00000001:1.0:1788574484.886638:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306089401
00000100:00000001:1.0:1788574484.886639:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306088633
00000100:00000001:1.0:1788574484.886641:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306089401
00000100:00000001:1.0:1788574484.886642:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574484.886643:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574484.886644:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574484.886647:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574484/real 1788574484]  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574484.886651:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574484.886652:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574484.886653:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574484.886655:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574484.886656:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574484.886657:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574484.886659:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574484.886664:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886668:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574484.886669:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886670:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574484.886671:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574484.886673:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886677:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574484.886679:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886683:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574484.886685:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886686:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574484.886687:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886688:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574484.886690:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574484.886691:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574484.886693:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574484.886695:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574484.886697:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574484.886706:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574484.886708:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886710:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574484.886712:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393138048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886715:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574484.886716:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574484.886718:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574484.886719:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574484.886720:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574484.886721:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574484.886722:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574484.886723:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:1.0:1788574484.886724:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574484.886725:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574484.886726:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:1.0:1788574484.886728:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574484.886729:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574484.886732:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393138176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574484.886736:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574484.886736:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574484.886737:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574484.886739:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574484.886739:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574484.886740:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574484.886741:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574484.886742:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:1.0:1788574484.886743:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574484.886744:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574484.886745:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:1.0:1788574484.886746:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574484.886747:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886749:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574484.886751:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574484.886752:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574484.886753:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574484.886754:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574485.845501:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306089401, at 4306089408
00000100:00000040:1.0:1788574485.845501:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306089401, at 4306089408
00000100:00000010:0.0:1788574485.845516:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9900.
00000100:00000010:1.0:1788574485.845519:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9060.
00000100:00000001:1.0:1788574490.005706:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574490.005711:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574490.005715:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574490.005724:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.005740:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574490.005744:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574490.005749:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574490.005752:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574490.005757:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574490.005763:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393138304:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574490.005771:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574490.005776:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574490.005779:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574490.005781:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574490.005787:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574490.005790:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574490.005800:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574490.005804:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574490.005814:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574490.005822:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393138304, portal 25
00000100:00000001:1.0:1788574490.005825:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574490.005829:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:1.0:1788574490.005835:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.005867:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574490.005882:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574490.005890:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574490.005896:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393138304, offset 0
00000400:00000200:1.0:1788574490.005911:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574490.005928:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574490.005940:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574490.005950:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574490.005956:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574490.005963:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574490.005968:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574490.005975:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330336:330336:256:4294967295] 192.168.202.110@tcp LPNI seq info [330293:330293:8:4294967295]
00000400:00000200:1.0:1788574490.005981:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574490.005996:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574490.006004:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574490.006012:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574490.006021:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574490.006027:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1788574490.006036:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574490.006053:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1788574490.006057:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574490.006060:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574490.006068:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574490.006077:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574490.006087:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574490.006090:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574490.006096:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574490.006102:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006115:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574490.006118:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574515 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006134:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574490.006135:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574490.006140:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574490.006143:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574490.006146:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306094521
00000100:00000010:1.0:1788574490.006150:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9060.
00000100:00000001:1.0:1788574490.006156:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574490.006160:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306094521
00000100:00000010:1.0:1788574490.006163:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9900.
00000100:00000001:1.0:1788574490.006165:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574490.006168:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574490.006169:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574490.006171:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574490.006174:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574490/real 1788574490]  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574490.006181:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574490.006183:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574490.006184:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574490.006186:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574490.006187:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574490.006189:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574490.006191:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574490.006197:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006202:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574490.006203:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006205:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574490.006206:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574490.006209:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006214:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574490.006217:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006222:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574490.006224:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006225:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574490.006225:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006227:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574490.006229:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574490.006231:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574490.006233:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574490.006235:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574490.006237:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574490.006243:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393138304:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574490.006246:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006248:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574490.006250:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393138304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006255:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574490.006256:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574490.006258:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574490.006260:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574490.006261:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574490.006262:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574490.006264:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574490.006265:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:1.0:1788574490.006268:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574490.006270:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574490.006271:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:1.0:1788574490.006273:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574490.006274:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006276:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006281:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574490.006283:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574490.006284:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574490.006285:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574490.006286:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574490.006289:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006294:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574490.006295:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574490.006296:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574490.006297:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574490.006299:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574490.006301:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138432:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574490.006302:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574490.006304:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574490.006305:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574490.006306:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574490.006308:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574490.006309:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574490.006311:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574490.006313:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574490.006315:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574490.006317:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393138432, portal 10
00000100:00000001:1.0:1788574490.006319:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574490.006320:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:1.0:1788574490.006324:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006328:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574490.006330:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574490.006332:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574490.006333:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393138432, offset 0
00000400:00000200:1.0:1788574490.006336:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574490.006339:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574490.006341:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574490.006344:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574490.006346:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574490.006347:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574490.006349:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574490.006351:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330337:330337:256:4294967295] 192.168.202.110@tcp LPNI seq info [330294:330294:8:4294967295]
00000400:00000200:1.0:1788574490.006354:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574490.006359:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574490.006361:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574490.006363:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574490.006366:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574490.006368:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1788574490.006371:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574490.006373:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1788574490.006375:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574490.006377:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574490.006378:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574490.006381:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574490.006383:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574490.006384:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574490.006386:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574490.006388:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006422:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574490.006427:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574515 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006434:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574490.006436:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574490.006440:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574490.006445:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574490.006449:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574490.006450:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306093607
00000100:00000001:1.0:1788574490.006452:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306094521
00000100:00000001:1.0:1788574490.006454:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306093607
00000100:00000001:1.0:1788574490.006456:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306094521
00000100:00000001:1.0:1788574490.006457:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574490.006459:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574490.006460:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574490.006463:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574490/real 1788574490]  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574490.006468:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574490.006469:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574490.006471:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574490.006472:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574490.006474:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574490.006475:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574490.006477:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574490.006483:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006487:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574490.006488:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006489:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574490.006490:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574490.006492:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006496:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574490.006499:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006504:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574490.006506:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006507:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574490.006508:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006509:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574490.006510:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574490.006512:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574490.006514:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574490.006515:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574490.006518:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574490.006523:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138432:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574490.006525:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574490.006527:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006532:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574490.006533:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574490.006534:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574490.006535:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574490.006536:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574490.006538:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138560:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574490.006540:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574490.006541:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574490.006543:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574490.006544:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574490.006546:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574490.006548:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574490.006550:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574490.006552:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574490.006554:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574490.006556:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393138560, portal 4
00000100:00000001:1.0:1788574490.006557:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574490.006558:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:1.0:1788574490.006563:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006567:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574490.006569:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574490.006570:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574490.006572:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393138560, offset 0
00000400:00000200:1.0:1788574490.006574:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574490.006577:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574490.006579:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574490.006581:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574490.006583:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574490.006584:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574490.006586:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574490.006588:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330338:330338:256:4294967295] 192.168.202.110@tcp LPNI seq info [330295:330295:8:4294967295]
00000400:00000200:1.0:1788574490.006591:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574490.006595:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574490.006598:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574490.006599:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574490.006602:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574490.006604:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1788574490.006607:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574490.006609:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1788574490.006611:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574490.006613:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574490.006615:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574490.006617:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574490.006619:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574490.006620:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574490.006622:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574490.006624:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006629:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574490.006631:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574515 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006636:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574490.006637:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574490.006639:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574490.006641:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574490.006644:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574490.006646:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306093753
00000100:00000001:1.0:1788574490.006649:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306094521
00000100:00000001:1.0:1788574490.006651:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306093753
00000100:00000001:1.0:1788574490.006653:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306094521
00000100:00000001:1.0:1788574490.006655:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574490.006657:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574490.006659:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574490.006663:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574490/real 1788574490]  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574490.006671:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574490.006673:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574490.006675:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574490.006677:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574490.006679:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574490.006682:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574490.006686:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574490.006694:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006703:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574490.006704:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006706:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574490.006707:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574490.006709:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006714:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574490.006716:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006720:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574490.006722:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006723:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574490.006723:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006725:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574490.006726:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574490.006727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574490.006729:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574490.006730:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574490.006733:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574490.006738:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138560:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574490.006741:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006742:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574490.006744:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393138432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006748:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574490.006750:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574490.006752:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574490.006753:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574490.006755:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574490.006756:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574490.006757:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574490.006758:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:1.0:1788574490.006760:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574490.006761:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574490.006762:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:1.0:1788574490.006764:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574490.006765:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006766:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574490.006768:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393138560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574515 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574490.006772:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574490.006773:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574490.006773:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574490.006775:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574490.006775:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574490.006776:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574490.006777:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574490.006778:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:1.0:1788574490.006779:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574490.006780:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574490.006781:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:1.0:1788574490.006782:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574490.006783:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006784:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574490.006787:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574490.006788:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574490.006789:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574490.006790:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574490.965510:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306094521, at 4306094528
00000100:00000040:1.0:1788574490.965510:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306094521, at 4306094528
00000100:00000010:0.0:1788574490.965526:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9060.
00000100:00000010:1.0:1788574490.965529:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9900.
00000100:00000001:1.0:1788574495.125679:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574495.125685:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574495.125690:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574495.125699:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.125716:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574495.125720:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574495.125725:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574495.125729:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574495.125734:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574495.125741:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393138688:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574495.125749:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574495.125754:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574495.125757:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574495.125759:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574495.125766:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574495.125769:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574495.125778:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574495.125783:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574495.125791:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574495.125799:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393138688, portal 25
00000100:00000001:1.0:1788574495.125802:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574495.125805:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:1.0:1788574495.125812:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.125827:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574495.125832:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574495.125837:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574495.125841:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393138688, offset 0
00000400:00000200:1.0:1788574495.125851:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574495.125859:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574495.125867:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574495.125883:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574495.125891:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574495.125896:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574495.125904:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574495.125908:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330339:330339:256:4294967295] 192.168.202.110@tcp LPNI seq info [330296:330296:8:4294967295]
00000400:00000200:1.0:1788574495.125918:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574495.125927:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574495.125934:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574495.125940:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574495.125951:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574495.125957:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1788574495.125967:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574495.125973:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1788574495.125982:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574495.125984:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574495.125990:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574495.125997:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574495.126003:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574495.126006:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574495.126012:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574495.126019:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126031:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574495.126036:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574520 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126046:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574495.126050:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574495.126056:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574495.126063:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574495.126067:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306099641
00000100:00000010:1.0:1788574495.126084:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9900.
00000100:00000001:1.0:1788574495.126092:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574495.126100:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306099641
00000100:00000010:1.0:1788574495.126104:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f96c0.
00000100:00000001:1.0:1788574495.126108:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574495.126111:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574495.126113:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574495.126115:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574495.126120:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574495/real 1788574495]  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574495.126127:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574495.126128:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574495.126130:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574495.126132:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574495.126133:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574495.126135:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574495.126137:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574495.126143:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126148:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574495.126149:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126151:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574495.126152:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574495.126154:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126159:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574495.126162:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126167:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574495.126169:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126170:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574495.126171:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126173:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574495.126174:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574495.126176:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574495.126179:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574495.126180:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574495.126183:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574495.126189:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393138688:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574495.126192:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126194:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574495.126195:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393138688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126199:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574495.126201:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574495.126203:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574495.126205:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574495.126206:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574495.126207:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574495.126209:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574495.126211:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:1.0:1788574495.126213:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574495.126214:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574495.126216:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:1.0:1788574495.126218:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574495.126218:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126220:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126225:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574495.126227:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574495.126228:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574495.126229:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574495.126230:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574495.126233:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126237:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574495.126239:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574495.126240:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574495.126241:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574495.126243:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574495.126245:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138816:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574495.126247:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574495.126248:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574495.126249:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574495.126250:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574495.126252:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574495.126253:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574495.126255:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574495.126257:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574495.126260:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574495.126261:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393138816, portal 10
00000100:00000001:1.0:1788574495.126263:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574495.126264:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:1.0:1788574495.126268:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126273:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574495.126274:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574495.126277:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574495.126279:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393138816, offset 0
00000400:00000200:1.0:1788574495.126284:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574495.126287:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574495.126291:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574495.126294:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574495.126297:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574495.126300:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574495.126302:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574495.126305:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330340:330340:256:4294967295] 192.168.202.110@tcp LPNI seq info [330297:330297:8:4294967295]
00000400:00000200:1.0:1788574495.126310:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574495.126316:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574495.126321:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574495.126325:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574495.126329:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574495.126332:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1788574495.126337:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574495.126340:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1788574495.126344:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574495.126346:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574495.126349:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574495.126352:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574495.126356:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574495.126358:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574495.126360:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574495.126364:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126373:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574495.126377:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574520 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126383:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574495.126385:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574495.126388:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574495.126413:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574495.126418:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574495.126420:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306098727
00000100:00000001:1.0:1788574495.126423:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306099641
00000100:00000001:1.0:1788574495.126426:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306098727
00000100:00000001:1.0:1788574495.126428:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306099641
00000100:00000001:1.0:1788574495.126430:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574495.126432:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574495.126434:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574495.126440:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574495/real 1788574495]  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574495.126450:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574495.126451:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574495.126453:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574495.126456:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574495.126458:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574495.126461:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574495.126465:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574495.126475:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126483:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574495.126484:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126486:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574495.126488:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574495.126491:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126498:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574495.126503:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126510:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574495.126513:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126515:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574495.126517:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126519:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574495.126521:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574495.126524:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574495.126528:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574495.126530:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574495.126535:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574495.126544:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138816:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574495.126548:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574495.126552:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126560:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574495.126561:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574495.126564:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574495.126566:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574495.126568:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574495.126571:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574495.126575:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574495.126577:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574495.126579:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574495.126580:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574495.126584:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574495.126586:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574495.126590:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574495.126593:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574495.126597:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574495.126600:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393138944, portal 4
00000100:00000001:1.0:1788574495.126602:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574495.126603:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:1.0:1788574495.126609:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126614:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574495.126615:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574495.126617:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574495.126619:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393138944, offset 0
00000400:00000200:1.0:1788574495.126622:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574495.126625:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574495.126628:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574495.126630:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574495.126632:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574495.126634:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574495.126635:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574495.126637:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330341:330341:256:4294967295] 192.168.202.110@tcp LPNI seq info [330298:330298:8:4294967295]
00000400:00000200:1.0:1788574495.126640:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574495.126645:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574495.126648:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574495.126650:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574495.126653:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574495.126655:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1788574495.126658:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574495.126660:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1788574495.126663:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574495.126664:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574495.126666:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574495.126668:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574495.126671:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574495.126672:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574495.126673:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574495.126675:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126681:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574495.126683:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574520 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126687:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574495.126688:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574495.126690:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574495.126692:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574495.126694:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574495.126696:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306098873
00000100:00000001:1.0:1788574495.126697:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306099641
00000100:00000001:1.0:1788574495.126699:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306098873
00000100:00000001:1.0:1788574495.126700:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306099641
00000100:00000001:1.0:1788574495.126702:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574495.126703:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574495.126704:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574495.126706:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574495/real 1788574495]  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574495.126711:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574495.126712:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574495.126713:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574495.126715:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574495.126716:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574495.126717:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574495.126719:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574495.126724:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126728:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574495.126729:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574495.126732:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574495.126734:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126738:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574495.126740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126744:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574495.126746:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126747:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574495.126748:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126749:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574495.126751:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574495.126753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574495.126754:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574495.126756:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574495.126758:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574495.126763:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393138944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574495.126765:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126766:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574495.126768:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393138816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126772:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574495.126773:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574495.126774:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574495.126776:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574495.126777:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574495.126778:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574495.126779:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574495.126781:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:1.0:1788574495.126783:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574495.126784:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574495.126785:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:1.0:1788574495.126787:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574495.126788:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126789:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574495.126791:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393138944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574520 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574495.126799:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574495.126800:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574495.126801:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574495.126802:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574495.126803:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574495.126804:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574495.126805:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574495.126806:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:1.0:1788574495.126807:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574495.126807:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574495.126808:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:1.0:1788574495.126810:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574495.126811:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126812:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574495.126814:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574495.126815:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574495.126817:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574495.126818:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574496.085475:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306099641, at 4306099648
00000100:00000040:1.0:1788574496.085478:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306099641, at 4306099648
00000100:00000010:0.0:1788574496.085490:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9900.
00000100:00000010:1.0:1788574496.085492:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f96c0.
00000100:00000001:1.0:1788574500.245746:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574500.245751:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574500.245755:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574500.245763:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.245776:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574500.245779:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574500.245784:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574500.245788:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574500.245791:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574500.245797:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393139072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574500.245803:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574500.245807:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574500.245810:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574500.245814:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574500.245820:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574500.245823:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574500.245832:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574500.245836:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574500.245843:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574500.245850:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393139072, portal 25
00000100:00000001:1.0:1788574500.245855:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574500.245859:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:1.0:1788574500.245870:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.245888:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574500.245894:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574500.245900:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574500.245905:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393139072, offset 0
00000400:00000200:1.0:1788574500.245915:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574500.245926:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574500.245934:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574500.245940:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574500.245945:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574500.245951:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574500.245956:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574500.245963:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330342:330342:256:4294967295] 192.168.202.110@tcp LPNI seq info [330299:330299:8:4294967295]
00000400:00000200:1.0:1788574500.245974:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574500.245987:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574500.245994:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574500.246001:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574500.246009:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574500.246013:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1788574500.246021:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574500.246025:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1788574500.246032:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574500.246037:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574500.246045:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574500.246056:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574500.246066:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574500.246070:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574500.246077:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574500.246087:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246117:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574500.246123:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574525 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246135:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574500.246140:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574500.246149:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574500.246155:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574500.246159:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306104761
00000100:00000010:1.0:1788574500.246166:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f96c0.
00000100:00000001:1.0:1788574500.246173:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574500.246187:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306104761
00000100:00000010:1.0:1788574500.246191:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f95a0.
00000100:00000001:1.0:1788574500.246195:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574500.246199:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574500.246204:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574500.246208:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574500.246214:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574500/real 1788574500]  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574500.246229:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574500.246231:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574500.246240:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574500.246245:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574500.246249:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574500.246254:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574500.246263:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574500.246286:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246313:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574500.246318:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246323:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574500.246327:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574500.246334:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246345:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574500.246351:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246359:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574500.246363:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246364:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574500.246366:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246368:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574500.246370:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574500.246373:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574500.246377:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574500.246380:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574500.246385:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574500.246420:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393139072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574500.246425:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246427:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574500.246431:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393139072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246444:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574500.246447:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574500.246450:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574500.246453:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574500.246455:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574500.246457:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574500.246459:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574500.246461:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:1.0:1788574500.246466:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574500.246468:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574500.246470:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:1.0:1788574500.246473:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574500.246475:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246477:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246484:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574500.246486:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574500.246489:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574500.246491:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574500.246492:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574500.246496:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246503:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574500.246504:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574500.246506:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574500.246508:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574500.246509:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574500.246511:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574500.246514:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574500.246516:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574500.246517:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574500.246518:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574500.246521:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574500.246522:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574500.246525:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574500.246528:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574500.246530:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574500.246533:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393139200, portal 10
00000100:00000001:1.0:1788574500.246534:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574500.246535:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:1.0:1788574500.246540:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246545:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574500.246547:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574500.246548:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574500.246550:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393139200, offset 0
00000400:00000200:1.0:1788574500.246552:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574500.246555:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574500.246558:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574500.246560:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574500.246562:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574500.246563:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574500.246565:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574500.246567:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330343:330343:256:4294967295] 192.168.202.110@tcp LPNI seq info [330300:330300:8:4294967295]
00000400:00000200:1.0:1788574500.246570:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574500.246575:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574500.246577:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574500.246580:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574500.246582:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574500.246584:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1788574500.246587:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574500.246589:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1788574500.246592:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574500.246593:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574500.246595:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574500.246598:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574500.246600:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574500.246601:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574500.246603:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574500.246606:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246611:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574500.246613:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574525 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246618:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574500.246619:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574500.246621:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574500.246623:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574500.246625:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574500.246627:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306103847
00000100:00000001:1.0:1788574500.246629:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306104761
00000100:00000001:1.0:1788574500.246630:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306103847
00000100:00000001:1.0:1788574500.246632:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306104761
00000100:00000001:1.0:1788574500.246633:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574500.246635:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574500.246636:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574500.246638:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574500/real 1788574500]  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574500.246643:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574500.246644:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574500.246645:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574500.246647:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574500.246648:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574500.246649:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574500.246651:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574500.246656:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246665:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574500.246666:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246667:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574500.246668:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574500.246670:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246674:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574500.246677:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246681:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574500.246682:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246683:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574500.246684:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246685:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574500.246687:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574500.246688:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574500.246690:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574500.246692:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574500.246694:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574500.246700:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574500.246702:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574500.246704:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246708:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574500.246709:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574500.246711:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574500.246712:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574500.246713:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574500.246715:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139328:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574500.246717:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574500.246719:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574500.246720:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574500.246721:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574500.246723:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758400.
02000000:00000001:1.0:1788574500.246724:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574500.246726:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574500.246728:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574500.246730:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574500.246732:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393139328, portal 4
00000100:00000001:1.0:1788574500.246733:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574500.246734:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:1.0:1788574500.246738:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246742:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574500.246744:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574500.246745:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574500.246746:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393139328, offset 0
00000400:00000200:1.0:1788574500.246749:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574500.246751:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574500.246753:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574500.246755:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574500.246756:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574500.246758:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574500.246760:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574500.246762:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330344:330344:256:4294967295] 192.168.202.110@tcp LPNI seq info [330301:330301:8:4294967295]
00000400:00000200:1.0:1788574500.246765:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574500.246769:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574500.246772:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574500.246774:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574500.246776:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574500.246778:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1788574500.246781:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574500.246783:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1788574500.246785:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574500.246787:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574500.246788:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574500.246791:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574500.246792:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574500.246793:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574500.246795:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574500.246797:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246802:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574500.246804:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574525 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246808:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574500.246809:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574500.246810:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574500.246812:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574500.246814:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574500.246815:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306103993
00000100:00000001:1.0:1788574500.246817:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306104761
00000100:00000001:1.0:1788574500.246818:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306103993
00000100:00000001:1.0:1788574500.246820:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306104761
00000100:00000001:1.0:1788574500.246821:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574500.246822:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574500.246823:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574500.246826:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574500/real 1788574500]  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574500.246831:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574500.246832:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574500.246833:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574500.246835:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574500.246836:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574500.246837:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574500.246839:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574500.246844:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246847:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574500.246848:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246849:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574500.246850:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574500.246853:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246859:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574500.246863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246870:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574500.246872:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246874:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574500.246875:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246876:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574500.246878:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574500.246881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574500.246884:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574500.246887:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574500.246891:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574500.246901:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139328:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574500.246904:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246906:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574500.246910:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393139200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246914:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574500.246916:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574500.246917:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574500.246919:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574500.246920:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574500.246921:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574500.246922:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574500.246924:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:1.0:1788574500.246925:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574500.246926:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574500.246927:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:1.0:1788574500.246929:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574500.246930:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246931:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574500.246933:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393139328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574525 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574500.246937:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574500.246938:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574500.246939:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758400.
02000000:00000001:1.0:1788574500.246941:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574500.246941:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574500.246942:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574500.246943:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574500.246944:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:1.0:1788574500.246945:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574500.246946:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574500.246947:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:1.0:1788574500.246949:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574500.246950:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246951:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574500.246953:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574500.246954:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574500.246955:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574500.246956:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574501.205484:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306104761, at 4306104768
00000100:00000010:0.0:1788574501.205497:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f96c0.
00000100:00000040:1.0:1788574501.205561:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306104761, at 4306104768
00000100:00000010:1.0:1788574501.205566:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f95a0.
00000100:00000001:1.0:1788574505.365795:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574505.365803:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574505.365808:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574505.365827:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.365867:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574505.365873:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574505.365884:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574505.365892:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574505.365900:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574505.365913:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393139456:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574505.365935:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574505.365945:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574505.365948:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574505.365952:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574505.365967:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758400.
02000000:00000001:1.0:1788574505.365975:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574505.365990:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574505.365998:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574505.366015:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574505.366023:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393139456, portal 25
00000100:00000001:1.0:1788574505.366031:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574505.366034:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:1.0:1788574505.366050:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366069:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574505.366077:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574505.366084:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574505.366087:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393139456, offset 0
00000400:00000200:1.0:1788574505.366093:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574505.366102:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574505.366118:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574505.366149:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574505.366157:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574505.366163:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574505.366175:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574505.366185:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330345:330345:256:4294967295] 192.168.202.110@tcp LPNI seq info [330302:330302:8:4294967295]
00000400:00000200:1.0:1788574505.366194:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574505.366219:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574505.366227:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574505.366241:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574505.366260:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574505.366266:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1788574505.366277:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574505.366280:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1788574505.366287:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574505.366292:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574505.366299:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574505.366308:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574505.366317:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574505.366322:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574505.366329:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574505.366338:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366378:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574505.366385:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366442:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574505.366444:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574505.366450:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574505.366454:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574505.366457:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306109882
00000100:00000010:1.0:1788574505.366462:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f95a0.
00000100:00000001:1.0:1788574505.366468:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574505.366474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306109882
00000100:00000010:1.0:1788574505.366476:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9d80.
00000100:00000001:1.0:1788574505.366479:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574505.366482:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574505.366484:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574505.366486:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574505.366490:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574505/real 1788574505]  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574505.366508:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574505.366510:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574505.366512:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574505.366514:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574505.366515:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574505.366517:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574505.366519:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574505.366525:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366529:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574505.366530:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.366532:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574505.366533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574505.366535:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366541:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574505.366544:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366549:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574505.366551:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.366552:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574505.366552:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.366554:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574505.366556:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574505.366558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574505.366561:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574505.366562:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574505.366565:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574505.366570:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393139456:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574505.366573:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.366575:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574505.366577:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393139456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366581:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574505.366583:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574505.366585:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758400.
02000000:00000001:1.0:1788574505.366587:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574505.366589:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574505.366590:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574505.366592:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574505.366594:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:1.0:1788574505.366596:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574505.366598:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574505.366600:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:1.0:1788574505.366603:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574505.366604:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.366605:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.366611:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574505.366612:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574505.366614:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574505.366615:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574505.366616:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574505.366619:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366624:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574505.366625:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574505.366627:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574505.366628:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574505.366630:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574505.366631:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139584:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574505.366633:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574505.366635:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574505.366636:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574505.366637:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574505.366639:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758400.
02000000:00000001:1.0:1788574505.366641:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574505.366643:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574505.366645:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574505.366647:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574505.366649:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393139584, portal 10
00000100:00000001:1.0:1788574505.366651:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574505.366652:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:1.0:1788574505.366656:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366660:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574505.366662:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574505.366664:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574505.366665:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393139584, offset 0
00000400:00000200:1.0:1788574505.366669:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574505.366674:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574505.366677:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574505.366680:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574505.366683:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574505.366685:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574505.366688:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574505.366691:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330346:330346:256:4294967295] 192.168.202.110@tcp LPNI seq info [330303:330303:8:4294967295]
00000400:00000200:1.0:1788574505.366696:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574505.366703:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574505.366708:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574505.366711:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574505.366715:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574505.366719:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1788574505.366724:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574505.366727:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1788574505.366731:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574505.366733:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574505.366736:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574505.366740:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574505.366743:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574505.366745:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574505.366748:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574505.366752:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366760:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574505.366763:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366767:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574505.366768:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574505.366772:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574505.366774:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574505.366777:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574505.366778:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306108967
00000100:00000001:1.0:1788574505.366780:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306109882
00000100:00000001:1.0:1788574505.366782:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306108967
00000100:00000001:1.0:1788574505.366784:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306109882
00000100:00000001:1.0:1788574505.366785:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574505.366786:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574505.366788:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574505.366790:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574505/real 1788574505]  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574505.366795:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574505.366796:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574505.366798:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574505.366799:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574505.366800:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574505.366801:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574505.366803:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574505.366809:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366814:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574505.366816:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.366818:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574505.366819:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574505.366823:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366830:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574505.366835:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366842:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574505.366845:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.366847:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574505.366848:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.366850:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574505.366853:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574505.366856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574505.366860:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574505.366862:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574505.366867:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574505.366875:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139584:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574505.366880:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574505.366884:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366892:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574505.366893:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574505.366896:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574505.366898:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574505.366900:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574505.366904:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139712:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574505.366908:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574505.366910:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574505.366912:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574505.366914:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574505.366918:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574505.366921:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574505.366925:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574505.366928:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000400:00000010:1.0:1788574505.366932:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00000200:1.0:1788574505.366936:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393139712, portal 4
00000100:00000001:1.0:1788574505.366938:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574505.366939:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:1.0:1788574505.366946:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.366952:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574505.366955:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574505.366957:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574505.366960:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393139712, offset 0
00000400:00000200:1.0:1788574505.366964:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574505.366969:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574505.366974:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574505.366977:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574505.366980:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574505.366982:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574505.366985:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574505.366989:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330347:330347:256:4294967295] 192.168.202.110@tcp LPNI seq info [330304:330304:8:4294967295]
00000400:00000200:1.0:1788574505.366993:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574505.367000:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574505.367005:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574505.367009:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574505.367013:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574505.367016:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1788574505.367019:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574505.367021:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1788574505.367024:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574505.367025:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574505.367027:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574505.367030:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574505.367032:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574505.367034:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574505.367036:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574505.367039:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.367044:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574505.367046:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.367050:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574505.367052:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574505.367054:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574505.367056:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574505.367058:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574505.367060:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306109113
00000100:00000001:1.0:1788574505.367062:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306109882
00000100:00000001:1.0:1788574505.367063:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306109113
00000100:00000001:1.0:1788574505.367065:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306109882
00000100:00000001:1.0:1788574505.367066:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574505.367068:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574505.367069:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574505.367071:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574505/real 1788574505]  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574505.367076:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574505.367077:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574505.367078:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574505.367079:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:1.0:1788574505.367080:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:1.0:1788574505.367081:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574505.367083:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574505.367089:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.367097:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574505.367098:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.367099:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574505.367100:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574505.367103:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.367107:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574505.367109:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.367113:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574505.367114:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.367115:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574505.367116:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.367117:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574505.367119:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574505.367120:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574505.367122:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574505.367124:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574505.367126:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574505.367149:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139712:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574505.367153:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.367156:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574505.367159:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393139584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.367165:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574505.367167:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574505.367169:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758400.
02000000:00000001:1.0:1788574505.367172:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574505.367173:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574505.367175:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574505.367177:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574505.367178:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:1.0:1788574505.367181:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574505.367182:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574505.367184:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:1.0:1788574505.367186:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574505.367187:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.367189:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574505.367192:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393139712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574530 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574505.367199:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574505.367200:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574505.367201:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574505.367204:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574505.367205:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574505.367206:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574505.367208:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574505.367210:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:1.0:1788574505.367212:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574505.367213:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574505.367215:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:1.0:1788574505.367217:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574505.367218:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.367220:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574505.367224:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574505.367225:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574505.367227:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574505.367228:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574506.325489:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306109882, at 4306109888
00000100:00000010:0.0:1788574506.325503:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f95a0.
00000100:00000040:1.0:1788574506.325573:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306109882, at 4306109888
00000100:00000010:1.0:1788574506.325578:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9d80.
00000100:00080000:0.0:1788574510.485521:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574510.485532:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574510.485535:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574510.485539:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574510.485544:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574510.485552:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11441
00000400:00000200:0.0:1788574510.485561:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574510.485567:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574510.485570:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574510.485573:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485575:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574510.485577:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485579:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574510.485582:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574510.485583:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574510.485585:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574510.485588:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574510.485590:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574510.485591:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574510.485593:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485594:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574510.485596:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485598:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574510.485600:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574510.485603:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485605:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574510.485607:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574510.485609:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574510.485611:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574510.485613:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574510.485615:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574510.485623:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788574510.485626:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574510.485629:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574510.485630:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574510.485632:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574510.485634:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574510.485638:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574510.485643:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788574510.485649:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574510.485656:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574510.485664:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574510.485667:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574510.485671:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574510.485688:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574510.485691:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574510.485693:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574510.485694:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574510.485696:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574510.485698:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574510.485700:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11441
00000400:00000200:0.0:1788574510.485703:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574510.485705:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574510.485706:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574510.485709:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485710:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574510.485712:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485714:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574510.485716:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574510.485717:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574510.485718:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574510.485719:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574510.485721:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574510.485722:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574510.485725:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485726:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574510.485727:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485729:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:1.0:1788574510.485730:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1788574510.485731:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:1.0:1788574510.485736:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574510.485740:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1788574510.485743:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485747:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574510.485750:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1788574510.485752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788574510.485754:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574510.485760:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574510.485768:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574510.485770:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:0.0:1788574510.485772:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1788574510.485774:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788574510.485776:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574510.485780:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574510.485780:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574510.485785:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:0.0:1788574510.485786:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574510.485790:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574510.485792:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574510.485793:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574510.485797:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574510.485800:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574510.485801:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:1.0:1788574510.485801:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393139840:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788574510.485804:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574510.485810:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:1.0:1788574510.485811:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1788574510.485814:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574510.485816:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574510.485820:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574510.485821:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1788574510.485824:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788574510.485830:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000010:1.0:1788574510.485832:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758400.
00000100:00000001:0.0:1788574510.485833:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574510.485837:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574510.485839:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574510.485848:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574510.485848:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788574510.485853:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1788574510.485855:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5de70.
00000100:00080000:0.0:1788574510.485856:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574510.485859:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574510.485862:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574510.485864:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:1.0:1788574510.485865:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2440.
00000100:00080000:0.0:1788574510.485866:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11441
00000400:00000200:0.0:1788574510.485872:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574510.485875:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000200:1.0:1788574510.485876:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393139840, portal 25
00000100:00080000:0.0:1788574510.485877:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1788574510.485882:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000040:0.0:1788574510.485883:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485888:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1788574510.485888:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523511296 : -130484186040320 : ffff895348713800)
00000100:00000040:0.0:1788574510.485892:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485895:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574510.485899:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574510.485900:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000040:1.0:1788574510.485900:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1788574510.485903:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574510.485906:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574510.485910:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574510.485916:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:1.0:1788574510.485918:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574510.485924:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:1.0:1788574510.485925:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574510.485930:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000001:0.0:1788574510.485931:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1788574510.485935:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393139840, offset 0
00000100:00000040:0.0:1788574510.485938:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485943:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:1.0:1788574510.485947:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:0.0:1788574510.485948:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574510.485955:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1788574510.485959:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574510.485960:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574510.485964:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574510.485967:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1788574510.485968:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1788574510.485970:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574510.485973:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574510.485975:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:1.0:1788574510.485976:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00000001:0.0:1788574510.485979:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574510.485982:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574510.485985:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000010:0.0:1788574510.485987:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574510.485990:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:1.0:1788574510.485992:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1788574510.485993:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574510.485996:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574510.485998:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788574510.485999:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574510.486002:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574510.486005:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574510.486008:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330348:330348:256:4294967295] 192.168.202.110@tcp LPNI seq info [330305:330305:8:4294967295]
02000000:00000010:0.0:1788574510.486010:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574510.486015:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574510.486020:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:0.0:1788574510.486025:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574510.486035:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574510.486038:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574510.486039:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1788574510.486044:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1788574510.486053:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788574510.486054:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574510.486059:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:1.0:1788574510.486062:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000100:00000040:0.0:1788574510.486063:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574510.486070:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11451)
00000800:00000200:1.0:1788574510.486075:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574510.486082:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574510.486100:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574510.486105:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574510.486109:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306115001
00000100:00000010:1.0:1788574510.486115:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9d80.
00000100:00000001:1.0:1788574510.486122:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574510.486133:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306115001
00000100:00000010:1.0:1788574510.486138:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f94e0.
00000100:00000001:1.0:1788574510.486142:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574510.486148:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574510.486151:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574510.486156:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574510.486173:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574510.486181:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574510.486184:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1788574510.486186:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574510.486188:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574510.486189:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574510.486195:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574510.486204:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574510.486206:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574510.486208:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574510.486210:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574510.486213:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574510.486216:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574510.486220:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574510.486223:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574510.486225:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574510.486226:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574510.486231:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574510.486233:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574510.486237:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574510.486241:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5df20.
00000400:00000010:1.0:1788574510.486246:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2550.
00000100:00000200:1.0:1788574510.486249:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393139968, portal 10
00000100:00000001:1.0:1788574510.486252:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574510.486254:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00000040:1.0:1788574510.486262:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574510.486271:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574510.486274:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574510.486276:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574510.486279:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393139968, offset 0
00000400:00000200:1.0:1788574510.486284:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574510.486289:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574510.486293:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574510.486296:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574510.486299:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574510.486302:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574510.486305:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574510.486308:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330349:330349:255:4294967295] 192.168.202.110@tcp LPNI seq info [330306:330306:7:4294967295]
00000400:00000200:1.0:1788574510.486313:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574510.486321:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574510.486326:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574510.486330:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cee00.
00000800:00000200:1.0:1788574510.486334:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574510.486339:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574510.486342:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574510.486346:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574510.486350:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574510.486352:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306114086
00000100:00000001:1.0:1788574510.486355:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306115001
00000100:00000001:1.0:1788574510.486357:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306114086
00000100:00000001:1.0:1788574510.486360:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306115001
00000100:00000001:1.0:1788574510.486363:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574510.486365:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574510.486368:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574510.486372:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574510.486386:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574510.486388:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574510.486416:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574510.486419:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574510.486422:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574510.486425:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574510.486429:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574510.486431:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574510.486433:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574510.486435:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574510.486439:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975a000.
02000000:00000001:1.0:1788574510.486441:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574510.486445:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574510.486447:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d790.
00000400:00000010:1.0:1788574510.486452:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2330.
00000100:00000200:1.0:1788574510.486456:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393140096, portal 4
00000100:00000001:1.0:1788574510.486457:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574510.486459:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000040:1.0:1788574510.486466:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574510.486474:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574510.486477:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574510.486480:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574510.486482:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393140096, offset 0
00000400:00000200:1.0:1788574510.486487:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574510.486491:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574510.486496:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574510.486499:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574510.486502:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574510.486504:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574510.486507:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574510.486511:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330350:330350:254:4294967295] 192.168.202.110@tcp LPNI seq info [330307:330307:6:4294967295]
00000400:00000200:1.0:1788574510.486516:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574510.486523:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574510.486528:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574510.486532:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574510.486536:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574510.486540:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574510.486543:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574510.486546:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574510.486550:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574510.486552:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306114233
00000100:00000001:1.0:1788574510.486555:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306115001
00000100:00000001:1.0:1788574510.486557:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306114233
00000100:00000001:1.0:1788574510.486560:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306115001
00000100:00000001:1.0:1788574510.486562:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574510.486565:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574510.486568:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574510.486570:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574510.486574:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574510.486577:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1788574510.486579:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574510.486580:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574510.486582:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574510.486584:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574511.445505:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306115001, at 4306115008
00000100:00000040:0.0:1788574511.445515:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306115001, at 4306115008
00000100:00000010:1.0:1788574511.445518:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f94e0.
00000100:00000010:0.0:1788574511.445531:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9d80.
00000400:00000100:1.0:1788574513.557715:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788574513.557769:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788574513.557783:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574513.557788:0:2352:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788574513.557793:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574513.557798:0:2352:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788574513.557803:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574513.557807:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574513.557813:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574513.557818:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574513.557824:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:1.0:1788574513.557827:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574513.557832:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574513.557841:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574513.557873:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574513.557879:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574535 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574513.557888:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574513.557891:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574513.557902:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574513.557905:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cee00.
00000400:00000200:1.0:1788574513.557909:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574513.557913:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574513.557917:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574513.557921:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:0.0:1788574513.557922:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574513.557925:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:0.0:1788574513.557928:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000200:1.0:1788574513.557933:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574513.557944:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1788574513.557948:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:1.0:1788574513.557949:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574535 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574513.557961:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574510/real 1788574513]  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574513.557962:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574513.557966:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574513.557977:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000400:00000001:0.0:1788574513.557980:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000800:00000010:1.0:1788574513.557981:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000001:0.0:1788574513.557984:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000200:1.0:1788574513.557987:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000010:0.0:1788574513.557990:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5de70.
00000400:00000200:1.0:1788574513.557994:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788574513.557997:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2440
00000400:00000010:0.0:1788574513.558001:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2440.
00000100:00000001:0.0:1788574513.558006:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000400:00000200:1.0:1788574513.558009:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000100:00000200:0.0:1788574513.558013:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788574513.558014:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574513.558021:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574513.558030:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574513.558033:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574513.558055:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1788574513.558056:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574513.558061:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574513.558063:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574535 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574513.558073:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574513.558075:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574513.558075:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574513.558080:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574513.558087:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558096:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574513.558103:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558112:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574513.558117:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558118:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574513.558120:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558123:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574513.558125:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574513.558130:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574513.558135:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574513.558138:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574513.558143:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574513.558152:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393139840:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574513.558158:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574513.558160:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574513.558164:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713800 x1875444393139840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558171:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574513.558174:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574513.558188:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758400.
02000000:00000001:0.0:1788574513.558193:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574513.558196:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574513.558198:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574513.558200:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574513.558203:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574513.558206:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574513.558209:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574513.558211:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713800.
00000100:00000001:0.0:1788574513.558215:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574513.558216:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558219:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574513.558221:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574513.558223:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574513.558224:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574513.558229:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574510/real 1788574513]  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574513.558237:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574513.558239:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574513.558241:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5df20.
00000400:00000200:0.0:1788574513.558244:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2550
00000400:00000010:0.0:1788574513.558246:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2550.
00000100:00000001:0.0:1788574513.558249:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574513.558251:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574513.558260:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558265:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574513.558267:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558269:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574513.558270:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574513.558274:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558281:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574513.558284:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558290:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574513.558293:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558294:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574513.558295:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558297:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574513.558299:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574513.558302:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574513.558305:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574513.558307:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574513.558317:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574513.558325:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393139968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574513.558329:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574513.558332:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574510/real 1788574513]  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574513.558340:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574513.558341:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574513.558343:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d790.
00000400:00000200:0.0:1788574513.558346:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2330
00000400:00000010:0.0:1788574513.558347:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2330.
00000100:00000001:0.0:1788574513.558349:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574513.558353:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574513.558360:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558367:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574513.558368:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558370:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574513.558371:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574513.558374:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558380:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574513.558384:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558452:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574513.558456:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558458:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574513.558459:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558461:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574513.558463:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574513.558466:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574513.558470:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574513.558473:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574513.558478:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574513.558486:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574513.558491:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558493:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574513.558496:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393139968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558503:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574513.558504:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574513.558507:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:0.0:1788574513.558511:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574513.558512:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574513.558513:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574513.558515:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574513.558517:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574513.558535:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574513.558537:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574513.558539:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:0.0:1788574513.558541:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574513.558542:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558544:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574513.558549:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393140096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574535 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574513.558562:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574513.558564:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574513.558565:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975a000.
02000000:00000001:0.0:1788574513.558568:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574513.558570:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574513.558571:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574513.558573:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574513.558575:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574513.558577:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574513.558579:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574513.558580:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:0.0:1788574513.558582:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574513.558584:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558586:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574513.558594:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574513.558596:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574513.558598:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574513.558600:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574515.606330:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574515.606336:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574515.606339:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574515.606347:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.606356:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574515.606359:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574515.606364:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574515.606367:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574515.606370:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574515.606375:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393140224:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574515.606380:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574515.606385:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574515.606389:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574515.606412:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574515.606419:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574515.606423:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574515.606430:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574515.606435:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574515.606443:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574515.606451:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393140224, portal 25
00000100:00000001:0.0:1788574515.606454:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574515.606456:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:0.0:1788574515.606464:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.606482:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574515.606485:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574515.606489:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574515.606493:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393140224, offset 0
00000400:00000200:0.0:1788574515.606501:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574515.606510:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574515.606516:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574515.606522:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574515.606527:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574515.606531:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574515.606534:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574515.606539:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330351:330351:256:4294967295] 192.168.202.110@tcp LPNI seq info [330308:330308:8:4294967295]
00000400:00000200:0.0:1788574515.606545:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574515.606556:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574515.606563:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574515.606569:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574515.606577:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574515.606581:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574515.606587:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574515.606590:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574515.606595:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574515.606598:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574515.606604:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574515.606609:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574515.606616:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574515.606619:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574515.606624:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574515.606628:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.606638:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574515.606643:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.606650:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574515.606652:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574515.606658:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574515.606663:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574515.606666:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306120122
00000100:00000010:0.0:1788574515.606672:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574515.606679:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574515.606685:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306120122
00000100:00000010:0.0:1788574515.606689:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7840.
00000100:00000001:0.0:1788574515.606692:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574515.606696:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574515.606699:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574515.606701:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574515.606707:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574515/real 1788574515]  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574515.606718:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574515.606720:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574515.606722:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574515.606725:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574515.606727:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574515.606729:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574515.606739:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574515.606750:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.606758:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574515.606761:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.606763:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574515.606765:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574515.606769:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.606777:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574515.606782:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.606790:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574515.606793:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.606795:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574515.606796:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.606799:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574515.606801:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574515.606804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574515.606808:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574515.606811:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574515.606816:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574515.606825:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393140224:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574515.606830:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574515.606834:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.606841:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574515.606842:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574515.606845:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574515.606848:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574515.606850:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574515.606853:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140352:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574515.606857:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574515.606859:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574515.606861:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574515.606863:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574515.606867:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574515.606869:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574515.606872:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574515.606875:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574515.606879:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574515.606883:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393140352, portal 10
00000100:00000001:0.0:1788574515.606885:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574515.606886:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:0.0:1788574515.606893:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.606899:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574515.606902:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574515.606904:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574515.606906:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393140352, offset 0
00000400:00000200:0.0:1788574515.606910:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574515.606921:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574515.606926:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574515.606929:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574515.606932:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574515.606935:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574515.606938:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574515.606941:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330352:330352:256:4294967295] 192.168.202.110@tcp LPNI seq info [330309:330309:8:4294967295]
00000400:00000200:0.0:1788574515.606946:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574515.606952:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574515.606957:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574515.606962:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574515.606966:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574515.606980:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574515.606985:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574515.606988:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574515.606992:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574515.606994:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574515.606997:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574515.607001:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574515.607004:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574515.607005:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574515.607008:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574515.607011:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607019:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574515.607023:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607029:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574515.607031:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574515.607034:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574515.607038:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574515.607042:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574515.607044:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306119207
00000100:00000001:0.0:1788574515.607047:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306120122
00000100:00000001:0.0:1788574515.607049:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306119207
00000100:00000001:0.0:1788574515.607051:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306120122
00000100:00000001:0.0:1788574515.607053:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574515.607056:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574515.607057:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574515.607061:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574515/real 1788574515]  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574515.607077:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574515.607078:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574515.607080:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574515.607083:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574515.607085:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574515.607087:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574515.607091:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574515.607100:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607107:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574515.607109:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607111:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574515.607112:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574515.607116:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607124:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574515.607128:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607136:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574515.607138:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607140:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574515.607141:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607143:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574515.607145:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574515.607148:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574515.607151:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574515.607154:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574515.607158:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574515.607167:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140352:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574515.607172:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574515.607175:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607182:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574515.607183:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574515.607186:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574515.607204:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574515.607207:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574515.607210:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140480:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574515.607213:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574515.607215:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574515.607216:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574515.607218:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574515.607220:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574515.607222:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574515.607225:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574515.607226:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574515.607229:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574515.607231:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393140480, portal 4
00000100:00000001:0.0:1788574515.607233:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574515.607233:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:0.0:1788574515.607238:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607245:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574515.607248:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574515.607250:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574515.607253:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393140480, offset 0
00000400:00000200:0.0:1788574515.607257:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574515.607261:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574515.607265:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574515.607268:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574515.607271:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574515.607274:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574515.607277:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574515.607280:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330353:330353:256:4294967295] 192.168.202.110@tcp LPNI seq info [330310:330310:8:4294967295]
00000400:00000200:0.0:1788574515.607284:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574515.607291:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574515.607296:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574515.607300:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574515.607304:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574515.607307:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788574515.607309:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574515.607311:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788574515.607314:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574515.607316:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574515.607318:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574515.607320:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574515.607323:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574515.607324:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574515.607326:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574515.607329:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607334:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574515.607336:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607341:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574515.607342:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574515.607344:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574515.607346:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574515.607348:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574515.607350:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306119353
00000100:00000001:0.0:1788574515.607352:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306120122
00000100:00000001:0.0:1788574515.607354:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306119353
00000100:00000001:0.0:1788574515.607355:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306120122
00000100:00000001:0.0:1788574515.607356:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574515.607358:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574515.607359:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574515.607361:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574515/real 1788574515]  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574515.607366:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574515.607368:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574515.607369:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574515.607370:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574515.607371:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574515.607373:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574515.607375:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574515.607380:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607384:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574515.607385:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607386:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574515.607387:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574515.607389:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607415:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574515.607418:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607423:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574515.607425:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607426:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574515.607427:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607428:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574515.607430:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574515.607432:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574515.607434:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574515.607435:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574515.607437:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574515.607443:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140480:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574515.607445:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607447:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574515.607449:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393140224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607453:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574515.607455:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574515.607457:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574515.607459:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574515.607460:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574515.607462:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574515.607463:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574515.607466:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:0.0:1788574515.607469:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574515.607471:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574515.607472:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:0.0:1788574515.607474:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574515.607475:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607476:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574515.607478:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393140352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607486:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574515.607487:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574515.607488:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574515.607490:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574515.607491:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574515.607492:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574515.607493:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574515.607493:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:0.0:1788574515.607495:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574515.607496:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574515.607496:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:0.0:1788574515.607498:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574515.607499:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607500:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574515.607501:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393140480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574515.607505:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574515.607506:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574515.607507:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574515.607508:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574515.607509:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574515.607509:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574515.607510:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574515.607511:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:0.0:1788574515.607512:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574515.607513:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574515.607514:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:0.0:1788574515.607516:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574515.607517:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607518:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574515.607524:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574515.607525:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574515.607527:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574515.607528:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574516.565487:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306120122, at 4306120128
00000100:00000040:1.0:1788574516.565487:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306120122, at 4306120128
00000100:00000010:1.0:1788574516.565504:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7840.
00000100:00000010:0.0:1788574516.565516:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574520.725725:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574520.725733:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574520.725739:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574520.725752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.725773:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574520.725776:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574520.725782:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574520.725788:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574520.725793:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574520.725799:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393140608:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574520.725806:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574520.725812:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574520.725815:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574520.725819:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574520.725826:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574520.725830:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574520.725840:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574520.725847:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574520.725858:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574520.725868:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393140608, portal 25
00000100:00000001:0.0:1788574520.725887:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574520.725890:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:0.0:1788574520.725900:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.725909:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574520.725915:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574520.725919:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574520.725926:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393140608, offset 0
00000400:00000200:0.0:1788574520.725936:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574520.725948:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574520.725959:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574520.725965:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574520.725971:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574520.725977:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574520.725981:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574520.725991:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330354:330354:256:4294967295] 192.168.202.110@tcp LPNI seq info [330311:330311:8:4294967295]
00000400:00000200:0.0:1788574520.726002:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574520.726020:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574520.726030:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574520.726036:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574520.726048:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574520.726052:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574520.726060:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574520.726063:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574520.726070:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574520.726073:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574520.726079:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574520.726087:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574520.726094:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574520.726097:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574520.726103:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574520.726109:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726131:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574520.726139:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726159:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574520.726162:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574520.726168:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574520.726172:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574520.726175:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306125241
00000100:00000010:0.0:1788574520.726179:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574520.726185:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574520.726190:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306125241
00000100:00000010:0.0:1788574520.726192:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7d80.
00000100:00000001:0.0:1788574520.726194:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574520.726197:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574520.726199:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574520.726201:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574520.726204:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574520/real 1788574520]  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574520.726212:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574520.726213:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574520.726215:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574520.726216:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574520.726247:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574520.726249:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574520.726252:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574520.726259:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726263:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574520.726265:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726267:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574520.726268:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574520.726271:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726275:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574520.726279:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726283:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574520.726285:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726286:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574520.726288:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726289:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574520.726291:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574520.726293:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574520.726296:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574520.726297:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574520.726300:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574520.726305:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393140608:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574520.726308:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726309:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574520.726312:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393140608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726316:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574520.726318:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574520.726320:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574520.726322:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574520.726323:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574520.726324:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574520.726326:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574520.726327:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788574520.726330:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574520.726332:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574520.726333:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:0.0:1788574520.726336:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574520.726337:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726338:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726344:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574520.726345:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574520.726347:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574520.726348:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574520.726349:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574520.726352:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726359:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574520.726361:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574520.726363:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574520.726365:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574520.726367:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574520.726370:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140736:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574520.726374:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574520.726376:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574520.726377:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574520.726379:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574520.726382:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574520.726384:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574520.726388:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574520.726414:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574520.726418:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574520.726420:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393140736, portal 10
00000100:00000001:0.0:1788574520.726421:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574520.726423:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:0.0:1788574520.726428:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726433:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574520.726434:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574520.726436:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574520.726437:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393140736, offset 0
00000400:00000200:0.0:1788574520.726440:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574520.726443:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574520.726446:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574520.726448:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574520.726450:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574520.726451:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574520.726453:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574520.726455:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330355:330355:256:4294967295] 192.168.202.110@tcp LPNI seq info [330312:330312:8:4294967295]
00000400:00000200:0.0:1788574520.726458:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574520.726463:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574520.726466:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574520.726469:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574520.726471:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574520.726473:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574520.726477:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574520.726478:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574520.726481:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574520.726482:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574520.726484:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574520.726487:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574520.726489:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574520.726490:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574520.726492:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574520.726494:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726500:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574520.726502:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726506:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574520.726507:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574520.726509:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574520.726512:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574520.726514:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574520.726515:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306124327
00000100:00000001:0.0:1788574520.726517:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306125241
00000100:00000001:0.0:1788574520.726519:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306124327
00000100:00000001:0.0:1788574520.726520:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306125241
00000100:00000001:0.0:1788574520.726522:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574520.726524:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574520.726525:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574520.726527:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574520/real 1788574520]  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574520.726533:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574520.726533:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574520.726535:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574520.726536:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574520.726537:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574520.726538:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574520.726540:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574520.726546:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726550:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574520.726551:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726552:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574520.726553:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574520.726555:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726559:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574520.726561:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726566:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574520.726567:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726568:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574520.726569:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726570:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574520.726572:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574520.726573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574520.726576:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574520.726577:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574520.726580:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574520.726585:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140736:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574520.726587:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574520.726590:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726593:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574520.726594:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574520.726596:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574520.726597:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574520.726598:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574520.726600:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574520.726602:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574520.726604:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574520.726605:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574520.726606:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574520.726608:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574520.726610:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574520.726612:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574520.726613:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574520.726615:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574520.726617:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393140864, portal 4
00000100:00000001:0.0:1788574520.726619:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574520.726619:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:0.0:1788574520.726623:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726627:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574520.726629:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574520.726631:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574520.726632:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393140864, offset 0
00000400:00000200:0.0:1788574520.726634:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574520.726637:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574520.726639:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574520.726641:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574520.726643:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574520.726644:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574520.726646:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574520.726648:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330356:330356:256:4294967295] 192.168.202.110@tcp LPNI seq info [330313:330313:8:4294967295]
00000400:00000200:0.0:1788574520.726651:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574520.726655:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574520.726658:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574520.726659:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574520.726662:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574520.726663:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574520.726666:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574520.726668:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574520.726670:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574520.726672:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574520.726674:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574520.726686:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574520.726688:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574520.726692:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574520.726694:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574520.726696:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726701:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574520.726703:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726708:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574520.726708:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574520.726711:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574520.726713:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574520.726715:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574520.726716:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306124473
00000100:00000001:0.0:1788574520.726718:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306125241
00000100:00000001:0.0:1788574520.726720:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306124473
00000100:00000001:0.0:1788574520.726721:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306125241
00000100:00000001:0.0:1788574520.726723:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574520.726724:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574520.726725:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574520.726728:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574520/real 1788574520]  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574520.726732:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574520.726733:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574520.726734:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574520.726735:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574520.726736:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574520.726737:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574520.726739:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574520.726744:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726748:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574520.726750:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574520.726752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574520.726754:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726758:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574520.726760:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726765:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574520.726767:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726768:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574520.726769:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726770:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574520.726771:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574520.726772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574520.726774:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574520.726776:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574520.726778:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574520.726783:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393140864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574520.726785:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726786:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574520.726788:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393140736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726792:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574520.726793:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574520.726794:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574520.726795:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574520.726796:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574520.726797:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574520.726798:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574520.726799:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788574520.726801:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574520.726802:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574520.726803:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:0.0:1788574520.726805:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574520.726806:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726807:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574520.726809:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393140864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574520.726814:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574520.726815:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574520.726816:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574520.726817:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574520.726817:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574520.726818:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574520.726819:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574520.726820:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574520.726821:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574520.726822:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574520.726823:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:0.0:1788574520.726824:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574520.726825:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726827:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574520.726830:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574520.726832:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574520.726833:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574520.726835:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574521.685485:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306125241, at 4306125248
00000100:00000040:1.0:1788574521.685486:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306125241, at 4306125248
00000100:00000010:1.0:1788574521.685502:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d80.
00000100:00000010:0.0:1788574521.685508:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000800:00000200:0.0:1788574521.749470:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788574521.749478:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788574521.749481:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788574521.749631:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1788574521.749639:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788574521.749646:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788574521.749652:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574521.749655:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.845714:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574525.845719:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574525.845724:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574525.845733:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.845771:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574525.845775:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574525.845781:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574525.845786:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574525.845792:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574525.845801:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393140992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574525.845808:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574525.845813:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574525.845817:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574525.845820:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574525.845826:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574525.845829:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574525.845838:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574525.845842:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574525.845849:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574525.845857:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393140992, portal 25
00000100:00000001:0.0:1788574525.845860:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574525.845862:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:0.0:1788574525.845868:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.845875:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574525.845881:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574525.845885:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574525.845889:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393140992, offset 0
00000400:00000200:0.0:1788574525.845897:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574525.845908:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574525.845916:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574525.845923:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574525.845931:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574525.845936:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574525.845942:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574525.845950:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330357:330357:256:4294967295] 192.168.202.110@tcp LPNI seq info [330314:330314:8:4294967295]
00000400:00000200:0.0:1788574525.845961:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574525.845976:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574525.845986:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574525.845992:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574525.846001:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574525.846006:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1788574525.846016:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574525.846021:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1788574525.846028:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574525.846033:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574525.846041:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574525.846049:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574525.846055:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574525.846058:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574525.846065:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574525.846071:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846084:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574525.846090:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846103:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574525.846108:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574525.846113:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574525.846119:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574525.846125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306130361
00000100:00000010:0.0:1788574525.846131:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574525.846139:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574525.846146:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306130361
00000100:00000010:0.0:1788574525.846153:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7660.
00000100:00000001:0.0:1788574525.846158:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574525.846163:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574525.846169:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574525.846172:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574525.846178:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574525/real 1788574525]  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574525.846192:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574525.846195:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574525.846199:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574525.846202:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574525.846204:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574525.846207:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574525.846213:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574525.846233:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846243:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574525.846269:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846273:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574525.846276:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574525.846287:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846299:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574525.846305:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846314:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574525.846317:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846319:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574525.846321:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846323:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574525.846325:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574525.846329:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574525.846333:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574525.846337:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574525.846341:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574525.846347:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393140992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574525.846351:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846352:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574525.846354:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393140992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846358:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574525.846360:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574525.846362:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574525.846365:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574525.846367:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574525.846368:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574525.846370:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574525.846372:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:0.0:1788574525.846374:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574525.846376:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574525.846377:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:0.0:1788574525.846379:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574525.846380:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846382:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846388:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574525.846406:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574525.846408:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574525.846409:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574525.846410:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574525.846413:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846418:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574525.846419:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574525.846421:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574525.846422:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574525.846424:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574525.846426:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574525.846428:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574525.846430:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574525.846431:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574525.846432:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574525.846435:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574525.846436:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574525.846439:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574525.846441:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574525.846443:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574525.846446:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393141120, portal 10
00000100:00000001:0.0:1788574525.846447:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574525.846448:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:0.0:1788574525.846453:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846457:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574525.846459:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574525.846460:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574525.846462:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393141120, offset 0
00000400:00000200:0.0:1788574525.846464:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574525.846467:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574525.846470:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574525.846472:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574525.846473:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574525.846475:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574525.846477:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574525.846479:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330358:330358:256:4294967295] 192.168.202.110@tcp LPNI seq info [330315:330315:8:4294967295]
00000400:00000200:0.0:1788574525.846482:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574525.846486:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574525.846489:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574525.846491:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574525.846494:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574525.846496:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1788574525.846499:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574525.846501:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1788574525.846503:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574525.846505:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574525.846507:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574525.846509:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574525.846511:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574525.846513:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574525.846514:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574525.846517:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846522:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574525.846524:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846529:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574525.846530:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574525.846531:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574525.846534:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574525.846536:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574525.846537:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306129447
00000100:00000001:0.0:1788574525.846539:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306130361
00000100:00000001:0.0:1788574525.846540:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306129447
00000100:00000001:0.0:1788574525.846541:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306130361
00000100:00000001:0.0:1788574525.846543:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574525.846544:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574525.846546:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574525.846548:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574525/real 1788574525]  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574525.846557:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574525.846558:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574525.846560:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574525.846561:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574525.846562:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574525.846563:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574525.846565:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574525.846570:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846574:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574525.846575:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846576:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574525.846577:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574525.846580:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846584:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574525.846586:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846590:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574525.846592:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846593:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574525.846594:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846595:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574525.846597:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574525.846598:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574525.846601:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574525.846602:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574525.846605:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574525.846610:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574525.846613:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574525.846615:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846619:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574525.846620:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574525.846621:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574525.846622:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574525.846623:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574525.846625:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141248:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574525.846627:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574525.846628:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574525.846629:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574525.846630:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574525.846632:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574525.846633:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574525.846635:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574525.846637:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574525.846639:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574525.846641:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393141248, portal 4
00000100:00000001:0.0:1788574525.846642:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574525.846643:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:0.0:1788574525.846647:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846652:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574525.846653:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574525.846655:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574525.846656:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393141248, offset 0
00000400:00000200:0.0:1788574525.846659:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574525.846661:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574525.846663:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574525.846665:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574525.846666:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574525.846668:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574525.846670:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574525.846672:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330359:330359:256:4294967295] 192.168.202.110@tcp LPNI seq info [330316:330316:8:4294967295]
00000400:00000200:0.0:1788574525.846674:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574525.846678:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574525.846681:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574525.846683:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574525.846685:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574525.846687:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1788574525.846689:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574525.846691:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1788574525.846694:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574525.846695:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574525.846697:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574525.846699:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574525.846701:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574525.846703:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574525.846704:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574525.846706:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846711:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574525.846713:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846717:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574525.846718:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574525.846720:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574525.846722:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574525.846724:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574525.846726:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306129593
00000100:00000001:0.0:1788574525.846728:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306130361
00000100:00000001:0.0:1788574525.846729:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306129593
00000100:00000001:0.0:1788574525.846731:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306130361
00000100:00000001:0.0:1788574525.846732:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574525.846733:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574525.846734:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574525.846736:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574525/real 1788574525]  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574525.846742:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574525.846743:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574525.846744:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574525.846747:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574525.846748:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574525.846750:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574525.846754:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574525.846763:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846769:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574525.846771:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846773:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574525.846774:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574525.846779:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846787:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574525.846791:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846798:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574525.846800:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846801:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574525.846802:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846803:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574525.846805:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574525.846806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574525.846808:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574525.846810:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574525.846812:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574525.846818:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141248:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574525.846820:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846821:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574525.846823:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393141120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846827:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574525.846828:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574525.846830:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574525.846831:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574525.846833:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574525.846833:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574525.846835:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574525.846836:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:0.0:1788574525.846838:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574525.846839:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574525.846840:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:0.0:1788574525.846842:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574525.846843:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846844:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574525.846847:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393141248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574525.846855:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574525.846855:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574525.846856:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788574525.846858:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574525.846859:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574525.846859:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574525.846860:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574525.846861:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:0.0:1788574525.846862:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574525.846863:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574525.846864:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:0.0:1788574525.846865:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574525.846866:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846867:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574525.846869:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574525.846870:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574525.846871:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574525.846872:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574526.805523:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306130361, at 4306130368
00000100:00000040:1.0:1788574526.805523:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306130361, at 4306130368
00000100:00000010:1.0:1788574526.805542:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7660.
00000100:00000010:0.0:1788574526.805545:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574530.965814:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574530.965821:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574530.965825:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574530.965834:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.965849:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574530.965852:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574530.965856:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574530.965860:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574530.965863:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574530.965869:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393141376:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574530.965876:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574530.965882:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574530.965887:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574530.965891:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574530.965898:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574530.965901:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574530.965910:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574530.965915:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574530.965925:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574530.965934:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393141376, portal 25
00000100:00000001:0.0:1788574530.965938:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574530.965940:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:0.0:1788574530.965947:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.965959:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574530.965963:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574530.965967:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574530.965970:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393141376, offset 0
00000400:00000200:0.0:1788574530.965978:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574530.965990:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574530.965999:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574530.966006:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574530.966013:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574530.966018:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574530.966023:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574530.966029:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330360:330360:256:4294967295] 192.168.202.110@tcp LPNI seq info [330317:330317:8:4294967295]
00000400:00000200:0.0:1788574530.966035:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574530.966050:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574530.966057:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574530.966065:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574530.966077:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574530.966083:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788574530.966090:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574530.966096:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1788574530.966101:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574530.966105:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574530.966114:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574530.966121:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574530.966126:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574530.966128:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574530.966133:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574530.966138:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966152:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574530.966159:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574555 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966168:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574530.966171:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574530.966179:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574530.966187:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574530.966194:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306135481
00000100:00000010:0.0:1788574530.966204:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574530.966216:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574530.966226:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306135481
00000100:00000010:0.0:1788574530.966232:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d70c0.
00000100:00000001:0.0:1788574530.966236:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574530.966242:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574530.966247:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574530.966249:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574530.966256:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574530/real 1788574530]  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574530.966273:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574530.966276:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574530.966291:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574530.966294:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574530.966295:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574530.966297:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574530.966301:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574530.966309:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966320:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574530.966322:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966325:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574530.966327:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574530.966333:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966344:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574530.966350:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966359:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574530.966362:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966364:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574530.966365:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966369:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574530.966372:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574530.966378:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574530.966383:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574530.966385:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574530.966411:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574530.966422:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393141376:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574530.966433:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966438:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574530.966446:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393141376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966481:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574530.966485:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574530.966489:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574530.966494:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574530.966497:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574530.966500:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574530.966505:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574530.966508:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788574530.966512:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574530.966515:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574530.966517:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:0.0:1788574530.966520:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574530.966521:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966523:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966530:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574530.966532:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574530.966534:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574530.966535:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574530.966537:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574530.966540:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966545:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574530.966546:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574530.966548:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574530.966550:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574530.966551:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574530.966553:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141504:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574530.966556:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574530.966557:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574530.966559:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574530.966560:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574530.966563:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574530.966564:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574530.966567:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574530.966570:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574530.966573:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574530.966575:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393141504, portal 10
00000100:00000001:0.0:1788574530.966577:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574530.966578:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:0.0:1788574530.966583:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966588:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574530.966589:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574530.966591:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574530.966593:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393141504, offset 0
00000400:00000200:0.0:1788574530.966595:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574530.966598:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574530.966601:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574530.966603:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574530.966605:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574530.966607:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574530.966609:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574530.966611:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330361:330361:256:4294967295] 192.168.202.110@tcp LPNI seq info [330318:330318:8:4294967295]
00000400:00000200:0.0:1788574530.966613:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574530.966618:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574530.966622:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574530.966624:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574530.966627:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574530.966629:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788574530.966632:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574530.966634:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1788574530.966637:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574530.966638:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574530.966641:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574530.966643:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574530.966646:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574530.966647:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574530.966649:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574530.966652:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966657:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574530.966659:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574555 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966664:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574530.966665:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574530.966667:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574530.966670:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574530.966672:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574530.966673:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306134567
00000100:00000001:0.0:1788574530.966675:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306135481
00000100:00000001:0.0:1788574530.966677:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306134567
00000100:00000001:0.0:1788574530.966679:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306135481
00000100:00000001:0.0:1788574530.966680:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574530.966682:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574530.966683:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574530.966685:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574530/real 1788574530]  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574530.966690:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574530.966691:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574530.966692:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574530.966693:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574530.966695:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574530.966696:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574530.966698:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574530.966704:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966712:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574530.966713:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966714:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574530.966715:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574530.966718:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966722:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574530.966724:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966728:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574530.966730:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574530.966732:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966733:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574530.966735:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574530.966737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574530.966739:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574530.966741:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574530.966743:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574530.966748:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141504:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574530.966750:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574530.966752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966757:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574530.966758:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574530.966759:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574530.966761:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574530.966762:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574530.966764:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141632:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574530.966767:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574530.966768:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574530.966768:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574530.966770:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574530.966772:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574530.966773:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574530.966775:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574530.966777:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574530.966779:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574530.966781:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393141632, portal 4
00000100:00000001:0.0:1788574530.966783:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574530.966784:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:0.0:1788574530.966788:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966792:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574530.966794:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574530.966795:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574530.966797:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393141632, offset 0
00000400:00000200:0.0:1788574530.966799:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574530.966801:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574530.966803:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574530.966805:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574530.966806:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574530.966808:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574530.966809:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574530.966811:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330362:330362:256:4294967295] 192.168.202.110@tcp LPNI seq info [330319:330319:8:4294967295]
00000400:00000200:0.0:1788574530.966814:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574530.966819:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574530.966822:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574530.966824:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574530.966827:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574530.966829:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788574530.966832:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574530.966834:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1788574530.966836:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574530.966838:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574530.966840:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574530.966842:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574530.966844:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574530.966845:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574530.966847:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574530.966849:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966854:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574530.966856:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574555 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966860:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574530.966861:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574530.966862:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574530.966864:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574530.966866:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574530.966867:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306134713
00000100:00000001:0.0:1788574530.966869:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306135481
00000100:00000001:0.0:1788574530.966871:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306134713
00000100:00000001:0.0:1788574530.966873:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306135481
00000100:00000001:0.0:1788574530.966874:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574530.966875:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574530.966876:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574530.966879:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574530/real 1788574530]  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574530.966884:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574530.966885:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574530.966887:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574530.966888:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574530.966889:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574530.966890:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574530.966892:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574530.966897:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966900:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574530.966901:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966902:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574530.966903:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574530.966905:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966909:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574530.966911:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966915:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574530.966917:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966918:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574530.966919:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966920:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574530.966921:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574530.966923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574530.966925:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574530.966927:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574530.966929:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574530.966935:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141632:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574530.966937:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966938:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574530.966940:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393141504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966944:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574530.966945:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574530.966946:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788574530.966947:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574530.966948:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574530.966949:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574530.966950:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574530.966951:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788574530.966953:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574530.966954:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574530.966955:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:0.0:1788574530.966956:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574530.966957:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966958:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574530.966960:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393141632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574555 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574530.966964:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574530.966965:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574530.966966:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788574530.966967:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574530.966968:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574530.966969:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574530.966970:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574530.966971:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:0.0:1788574530.966972:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574530.966973:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574530.966974:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:0.0:1788574530.966976:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574530.966977:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966978:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574530.966980:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574530.966981:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574530.966982:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574530.966983:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574531.925533:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306135481, at 4306135488
00000100:00000040:0.0:1788574531.925538:0:216226:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306135481, at 4306135488
00000100:00000010:1.0:1788574531.925555:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d70c0.
00000100:00000010:0.0:1788574531.925558:0:216226:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00080000:0.0:1788574536.085500:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574536.085508:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574536.085510:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574536.085514:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574536.085517:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574536.085523:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11467
00000400:00000200:0.0:1788574536.085531:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574536.085536:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574536.085538:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574536.085541:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085543:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574536.085545:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085547:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574536.085549:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574536.085551:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574536.085553:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574536.085555:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574536.085557:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574536.085558:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574536.085560:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085562:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574536.085563:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085565:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574536.085567:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574536.085569:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085571:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574536.085574:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574536.085576:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574536.085578:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574536.085580:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574536.085582:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574536.085589:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574536.085591:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574536.085594:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574536.085595:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574536.085597:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574536.085599:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574536.085602:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574536.085610:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574536.085616:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574536.085627:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574536.085639:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574536.085644:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574536.085650:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574536.085672:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574536.085676:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574536.085679:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574536.085681:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574536.085683:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574536.085686:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574536.085689:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11467
00000400:00000200:0.0:1788574536.085694:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574536.085699:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574536.085701:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574536.085705:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085707:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574536.085710:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085712:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574536.085714:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574536.085715:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:1.0:1788574536.085715:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788574536.085719:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574536.085722:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:1.0:1788574536.085722:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788574536.085724:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574536.085727:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:1.0:1788574536.085728:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574536.085730:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085732:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574536.085736:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085738:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574536.085740:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:1.0:1788574536.085740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788574536.085746:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085750:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574536.085754:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574536.085760:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574536.085762:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788574536.085764:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.085766:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574536.085767:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574536.085771:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1788574536.085773:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:0.0:1788574536.085774:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574536.085778:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574536.085779:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:0.0:1788574536.085781:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
02000000:00000001:1.0:1788574536.085785:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788574536.085786:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574536.085791:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574536.085794:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574536.085796:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:1.0:1788574536.085796:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393141760:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788574536.085799:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574536.085802:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.085805:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:0.0:1788574536.085807:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
02000000:00000001:1.0:1788574536.085811:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1788574536.085812:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574536.085815:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574536.085820:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1788574536.085820:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574536.085827:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759800.
00000100:00000001:0.0:1788574536.085828:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1788574536.085832:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574536.085833:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574536.085839:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574536.085843:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574536.085849:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000100:00000001:0.0:1788574536.085850:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574536.085855:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574536.085857:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574536.085858:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574536.085860:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1788574536.085860:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000001:0.0:1788574536.085866:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574536.085868:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11467
00000100:00000200:1.0:1788574536.085871:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393141760, portal 25
00000400:00000200:0.0:1788574536.085872:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574536.085877:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1788574536.085877:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1788574536.085882:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1788574536.085882:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000040:1.0:1788574536.085892:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574561 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574536.085896:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085901:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574536.085904:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085909:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574536.085914:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574536.085916:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:1.0:1788574536.085916:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:0.0:1788574536.085920:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574536.085921:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574536.085924:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574536.085926:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788574536.085928:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574536.085932:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:1.0:1788574536.085932:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000001:0.0:1788574536.085935:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574536.085938:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085940:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:1.0:1788574536.085941:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393141760, offset 0
00000020:00000040:0.0:1788574536.085942:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574536.085946:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574536.085947:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574536.085949:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574536.085951:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574536.085954:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574536.085955:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574536.085957:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574536.085960:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574536.085963:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574536.085964:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000010:0.0:1788574536.085967:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788574536.085970:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574536.085971:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574536.085973:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574536.085977:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574536.085978:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574536.085980:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574536.085983:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574536.085986:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574536.085988:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574536.085990:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:0.0:1788574536.085994:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1788574536.086001:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788574536.086002:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574536.086006:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574536.086011:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:0.0:1788574536.086012:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574536.086021:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788574536.086022:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574536.086027:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1788574536.086029:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:0.0:1788574536.086031:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574536.086037:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11477)
00000400:00000200:1.0:1788574536.086038:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330363:330363:256:4294967295] 192.168.202.110@tcp LPNI seq info [330320:330320:8:4294967295]
00000400:00000200:1.0:1788574536.086046:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574536.086056:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574536.086063:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574536.086071:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574536.086078:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574536.086082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1788574536.086089:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574536.086092:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1788574536.086097:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574536.086100:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574536.086107:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574536.086112:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574536.086118:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574536.086120:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574536.086125:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574536.086129:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574561 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086141:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574536.086145:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574561 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086153:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.086155:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574536.086161:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574536.086165:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574536.086168:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306140601
00000100:00000010:1.0:1788574536.086174:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f94e0.
00000100:00000001:1.0:1788574536.086180:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574536.086186:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306140601
00000100:00000010:1.0:1788574536.086201:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9f60.
00000100:00000001:1.0:1788574536.086205:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574536.086214:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574536.086217:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.086220:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574536.086225:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574536/real 1788574536]  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574536.086236:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574536.086238:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574536.086240:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574536.086243:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574536.086245:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574536.086248:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574536.086253:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574536.086263:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086271:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574536.086273:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.086275:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574536.086278:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574536.086282:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086290:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574536.086295:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086304:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574536.086321:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.086323:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574536.086325:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.086328:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574536.086330:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574536.086333:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574536.086338:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574536.086341:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574536.086346:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574536.086356:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393141760:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574536.086362:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.086364:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574536.086368:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393141760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086375:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574536.086377:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574536.086380:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759800.
02000000:00000001:1.0:1788574536.086384:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574536.086386:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574536.086387:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574536.086390:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574536.086421:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:1.0:1788574536.086426:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574536.086429:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574536.086430:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:1.0:1788574536.086434:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574536.086436:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.086438:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.086446:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574536.086449:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.086451:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574536.086452:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574536.086454:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574536.086458:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086466:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574536.086468:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574536.086470:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574536.086473:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574536.086475:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574536.086478:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574536.086482:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574536.086485:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574536.086486:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574536.086488:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574536.086492:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759800.
02000000:00000001:1.0:1788574536.086494:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574536.086499:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574536.086502:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574536.086506:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574536.086509:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393141888, portal 10
00000100:00000001:1.0:1788574536.086512:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574536.086513:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00000040:1.0:1788574536.086521:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574561 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086529:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574536.086532:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574536.086535:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574536.086537:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393141888, offset 0
00000400:00000200:1.0:1788574536.086542:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574536.086546:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574536.086551:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574536.086554:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574536.086557:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574536.086560:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574536.086562:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574536.086566:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330364:330364:256:4294967295] 192.168.202.110@tcp LPNI seq info [330321:330321:8:4294967295]
00000400:00000200:1.0:1788574536.086571:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574536.086578:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574536.086583:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574536.086587:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574536.086592:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574536.086595:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1788574536.086600:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574536.086603:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1788574536.086608:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574536.086610:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574536.086613:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574536.086617:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574536.086620:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574536.086622:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574536.086625:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574536.086629:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574561 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086638:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574536.086642:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574561 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086649:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.086650:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574536.086654:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574536.086658:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574536.086662:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574536.086664:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306139687
00000100:00000001:1.0:1788574536.086667:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306140601
00000100:00000001:1.0:1788574536.086669:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306139687
00000100:00000001:1.0:1788574536.086672:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306140601
00000100:00000001:1.0:1788574536.086674:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574536.086677:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.086679:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574536.086684:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574536/real 1788574536]  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574536.086693:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574536.086695:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574536.086697:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574536.086700:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574536.086701:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574536.086704:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574536.086708:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574536.086718:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086726:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574536.086727:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.086729:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574536.086731:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574536.086735:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086742:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574536.086746:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086754:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574536.086757:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.086758:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574536.086760:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.086762:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574536.086764:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574536.086767:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574536.086772:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574536.086774:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574536.086779:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574536.086787:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393141888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574536.086792:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574536.086796:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086803:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574536.086805:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574536.086807:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574536.086809:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574536.086811:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574536.086814:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574536.086818:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574536.086820:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574536.086822:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574536.086824:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574536.086827:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758000.
02000000:00000001:1.0:1788574536.086829:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574536.086833:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574536.086836:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574536.086840:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574536.086843:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393142016, portal 4
00000100:00000001:1.0:1788574536.086845:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574536.086846:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523511296 : -130484186040320 : ffff895348713800)
00000100:00000040:1.0:1788574536.086853:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574561 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086860:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574536.086864:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574536.086866:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574536.086869:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393142016, offset 0
00000400:00000200:1.0:1788574536.086873:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574536.086878:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574536.086882:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574536.086885:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574536.086888:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574536.086890:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574536.086893:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574536.086896:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330365:330365:256:4294967295] 192.168.202.110@tcp LPNI seq info [330322:330322:8:4294967295]
00000400:00000200:1.0:1788574536.086901:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574536.086908:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574536.086913:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574536.086917:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574536.086921:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574536.086924:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1788574536.086929:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574536.086932:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1788574536.086936:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574536.086938:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574536.086941:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574536.086945:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574536.086948:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574536.086950:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574536.086953:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574536.086957:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574561 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086965:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574536.086969:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574561 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.086976:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.086978:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574536.086982:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574536.086986:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574536.086989:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574536.086991:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306139833
00000100:00000001:1.0:1788574536.086994:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306140601
00000100:00000001:1.0:1788574536.086996:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306139833
00000100:00000001:1.0:1788574536.086998:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306140601
00000100:00000001:1.0:1788574536.087000:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574536.087002:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.087004:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574536.087008:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574536/real 1788574536]  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574536.087017:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574536.087018:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574536.087021:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574536.087023:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574536.087025:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574536.087027:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574536.087031:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574536.087040:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.087047:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574536.087049:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.087051:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574536.087053:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574536.087057:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.087064:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574536.087069:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.087077:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574536.087079:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.087081:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574536.087083:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.087085:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574536.087087:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574536.087090:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574536.087093:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574536.087096:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574536.087100:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574536.087109:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574536.087113:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.087115:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574536.087118:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393141888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.087126:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574536.087128:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574536.087130:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759800.
02000000:00000001:1.0:1788574536.087133:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574536.087134:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574536.087136:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574536.087138:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574536.087140:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:1.0:1788574536.087143:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574536.087144:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574536.087146:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:1.0:1788574536.087148:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574536.087150:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.087152:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574536.087156:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713800 x1875444393142016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574561 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574536.087163:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574536.087164:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574536.087166:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758000.
02000000:00000001:1.0:1788574536.087169:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574536.087170:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574536.087171:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574536.087173:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574536.087175:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:1.0:1788574536.087177:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574536.087179:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574536.087180:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713800.
00000100:00000001:1.0:1788574536.087182:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574536.087184:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.087186:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574536.087190:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574536.087191:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574536.087193:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574536.087195:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574537.045465:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306140601, at 4306140608
00000100:00000040:0.0:1788574537.045489:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306140601, at 4306140608
00000100:00000010:1.0:1788574537.045493:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9f60.
00000100:00000010:0.0:1788574537.045497:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f94e0.
00000100:00000001:1.0:1788574541.205688:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574541.205692:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574541.205695:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574541.205704:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.205724:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574541.205727:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574541.205732:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574541.205735:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574541.205740:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574541.205748:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393142144:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574541.205754:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574541.205759:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574541.205762:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574541.205765:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574541.205773:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759800.
02000000:00000001:1.0:1788574541.205776:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574541.205785:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574541.205790:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574541.205799:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574541.205806:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393142144, portal 25
00000100:00000001:1.0:1788574541.205809:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574541.205813:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000040:1.0:1788574541.205821:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.205828:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574541.205833:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574541.205836:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574541.205840:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393142144, offset 0
00000400:00000200:1.0:1788574541.205849:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574541.205860:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574541.205868:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574541.205876:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574541.205881:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574541.205887:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574541.205892:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574541.205898:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330366:330366:256:4294967295] 192.168.202.110@tcp LPNI seq info [330323:330323:8:4294967295]
00000400:00000200:1.0:1788574541.205907:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574541.205925:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574541.205935:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574541.205942:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574541.205953:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574541.205960:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1788574541.205971:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574541.205977:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1788574541.205984:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574541.205987:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574541.205994:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574541.206001:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574541.206007:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574541.206010:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574541.206015:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574541.206020:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206033:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574541.206037:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206049:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574541.206051:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574541.206056:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574541.206062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574541.206065:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306145721
00000100:00000010:1.0:1788574541.206069:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9f60.
00000100:00000001:1.0:1788574541.206076:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574541.206082:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306145721
00000100:00000010:1.0:1788574541.206087:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9240.
00000100:00000001:1.0:1788574541.206092:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574541.206096:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574541.206099:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574541.206101:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574541.206107:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574541/real 1788574541]  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574541.206118:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574541.206121:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574541.206124:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574541.206129:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574541.206134:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574541.206138:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574541.206145:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574541.206173:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206188:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574541.206191:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206195:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574541.206196:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574541.206199:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206209:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574541.206215:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206222:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574541.206227:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206228:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574541.206230:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206232:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574541.206234:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574541.206237:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574541.206241:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574541.206244:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574541.206248:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574541.206258:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393142144:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574541.206263:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206266:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574541.206269:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393142144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206283:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574541.206285:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574541.206288:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759800.
02000000:00000001:1.0:1788574541.206291:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574541.206293:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574541.206294:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574541.206297:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574541.206299:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:1.0:1788574541.206303:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574541.206305:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574541.206306:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:1.0:1788574541.206308:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574541.206309:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206311:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206317:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574541.206319:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574541.206320:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574541.206322:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574541.206323:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574541.206326:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206331:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574541.206332:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574541.206334:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574541.206347:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574541.206349:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574541.206352:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142272:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574541.206354:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574541.206356:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574541.206357:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574541.206359:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574541.206361:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759800.
02000000:00000001:1.0:1788574541.206362:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574541.206365:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574541.206367:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574541.206370:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574541.206372:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393142272, portal 10
00000100:00000001:1.0:1788574541.206374:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574541.206375:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:1.0:1788574541.206380:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206384:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574541.206386:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574541.206387:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574541.206389:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393142272, offset 0
00000400:00000200:1.0:1788574541.206421:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574541.206426:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574541.206430:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574541.206433:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574541.206436:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574541.206438:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574541.206441:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574541.206444:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330367:330367:256:4294967295] 192.168.202.110@tcp LPNI seq info [330324:330324:8:4294967295]
00000400:00000200:1.0:1788574541.206448:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574541.206456:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574541.206461:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574541.206465:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574541.206480:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574541.206483:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1788574541.206488:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574541.206491:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1788574541.206495:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574541.206497:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574541.206501:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574541.206505:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574541.206508:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574541.206510:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574541.206513:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574541.206517:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206526:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574541.206528:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206533:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574541.206534:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574541.206537:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574541.206540:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574541.206542:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574541.206544:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306144807
00000100:00000001:1.0:1788574541.206547:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306145721
00000100:00000001:1.0:1788574541.206548:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306144807
00000100:00000001:1.0:1788574541.206549:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306145721
00000100:00000001:1.0:1788574541.206551:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574541.206552:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574541.206554:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574541.206556:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574541/real 1788574541]  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574541.206561:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574541.206562:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574541.206563:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574541.206565:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574541.206566:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574541.206567:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574541.206569:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574541.206575:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206583:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574541.206584:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206585:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574541.206586:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574541.206589:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206593:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574541.206595:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206599:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574541.206601:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206602:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574541.206602:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206603:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574541.206605:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574541.206606:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574541.206608:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574541.206610:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574541.206612:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574541.206617:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142272:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574541.206620:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574541.206627:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206632:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574541.206633:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574541.206635:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574541.206636:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574541.206638:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574541.206640:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142400:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574541.206642:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574541.206643:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574541.206645:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574541.206646:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574541.206649:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758000.
02000000:00000001:1.0:1788574541.206650:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574541.206652:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574541.206654:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574541.206656:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574541.206658:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393142400, portal 4
00000100:00000001:1.0:1788574541.206660:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574541.206661:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:1.0:1788574541.206665:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206669:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574541.206671:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574541.206672:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574541.206673:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393142400, offset 0
00000400:00000200:1.0:1788574541.206676:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574541.206679:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574541.206681:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574541.206683:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574541.206684:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574541.206686:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574541.206688:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574541.206690:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330368:330368:256:4294967295] 192.168.202.110@tcp LPNI seq info [330325:330325:8:4294967295]
00000400:00000200:1.0:1788574541.206692:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574541.206697:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574541.206701:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574541.206703:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574541.206706:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574541.206708:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1788574541.206710:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574541.206712:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1788574541.206715:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574541.206716:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574541.206718:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574541.206721:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574541.206723:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574541.206724:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574541.206725:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574541.206727:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206732:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574541.206734:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206739:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574541.206740:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574541.206742:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574541.206744:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574541.206745:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574541.206747:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306144953
00000100:00000001:1.0:1788574541.206748:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306145721
00000100:00000001:1.0:1788574541.206750:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306144953
00000100:00000001:1.0:1788574541.206751:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306145721
00000100:00000001:1.0:1788574541.206753:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574541.206754:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574541.206755:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574541.206757:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574541/real 1788574541]  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574541.206763:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574541.206763:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574541.206765:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574541.206766:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574541.206767:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574541.206768:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574541.206770:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574541.206775:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206779:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574541.206780:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206781:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574541.206783:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574541.206786:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206793:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574541.206798:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206804:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574541.206806:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206808:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574541.206809:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206811:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574541.206813:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574541.206815:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574541.206819:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574541.206822:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574541.206827:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574541.206836:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142400:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574541.206840:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206842:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574541.206846:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393142272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206853:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574541.206855:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574541.206857:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759800.
02000000:00000001:1.0:1788574541.206860:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574541.206861:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574541.206863:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574541.206865:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574541.206867:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:1.0:1788574541.206869:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574541.206871:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574541.206873:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:1.0:1788574541.206875:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574541.206877:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206879:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574541.206883:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393142400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574566 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574541.206890:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574541.206892:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574541.206894:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758000.
02000000:00000001:1.0:1788574541.206896:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574541.206898:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574541.206899:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574541.206902:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574541.206903:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:1.0:1788574541.206906:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574541.206908:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574541.206909:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:1.0:1788574541.206912:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574541.206913:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206915:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574541.206919:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574541.206921:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574541.206923:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574541.206925:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574542.165499:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306145721, at 4306145728
00000100:00000040:1.0:1788574542.165499:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306145721, at 4306145728
00000100:00000010:0.0:1788574542.165514:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9f60.
00000100:00000010:1.0:1788574542.165520:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9240.
00000100:00000001:1.0:1788574546.325746:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574546.325752:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574546.325756:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574546.325766:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.325788:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574546.325791:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574546.325794:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574546.325798:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574546.325804:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574546.325811:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393142528:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574546.325819:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574546.325825:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574546.325830:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574546.325834:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574546.325841:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758000.
02000000:00000001:1.0:1788574546.325845:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574546.325853:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574546.325866:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574546.325876:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574546.325886:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393142528, portal 25
00000100:00000001:1.0:1788574546.325891:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574546.325894:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:1.0:1788574546.325903:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.325914:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574546.325920:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574546.325930:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574546.325935:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393142528, offset 0
00000400:00000200:1.0:1788574546.325945:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574546.325957:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574546.325966:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574546.325971:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574546.325975:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574546.325982:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574546.325992:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574546.326004:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330369:330369:256:4294967295] 192.168.202.110@tcp LPNI seq info [330326:330326:8:4294967295]
00000400:00000200:1.0:1788574546.326026:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574546.326042:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574546.326052:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574546.326061:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574546.326073:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574546.326078:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1788574546.326092:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574546.326096:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1788574546.326105:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574546.326107:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574546.326114:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574546.326122:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574546.326126:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574546.326128:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574546.326134:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574546.326140:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326154:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574546.326161:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326186:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574546.326189:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574546.326197:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574546.326205:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574546.326208:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306150841
00000100:00000010:1.0:1788574546.326212:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9240.
00000100:00000001:1.0:1788574546.326218:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574546.326231:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306150841
00000100:00000010:1.0:1788574546.326234:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9b40.
00000100:00000001:1.0:1788574546.326236:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574546.326238:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574546.326240:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574546.326242:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574546.326246:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574546/real 1788574546]  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574546.326253:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574546.326255:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574546.326257:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574546.326260:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574546.326262:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574546.326264:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574546.326266:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574546.326272:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326277:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574546.326278:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.326280:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574546.326281:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574546.326285:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326294:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574546.326299:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326307:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574546.326310:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.326312:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574546.326313:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.326316:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574546.326318:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574546.326321:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574546.326325:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574546.326328:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574546.326333:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574546.326342:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393142528:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574546.326347:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.326350:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574546.326353:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393142528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326360:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574546.326362:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574546.326379:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758000.
02000000:00000001:1.0:1788574546.326383:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574546.326386:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574546.326388:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574546.326390:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574546.326420:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:1.0:1788574546.326425:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574546.326427:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574546.326430:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:1.0:1788574546.326433:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574546.326435:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.326438:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.326452:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574546.326454:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574546.326456:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574546.326458:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574546.326460:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574546.326465:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326472:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574546.326474:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574546.326476:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574546.326478:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574546.326481:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574546.326484:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142656:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574546.326488:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574546.326490:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574546.326492:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574546.326494:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574546.326498:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758000.
02000000:00000001:1.0:1788574546.326500:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574546.326504:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574546.326507:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574546.326511:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574546.326515:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393142656, portal 10
00000100:00000001:1.0:1788574546.326527:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574546.326529:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:1.0:1788574546.326536:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326544:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574546.326546:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574546.326550:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574546.326552:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393142656, offset 0
00000400:00000200:1.0:1788574546.326557:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574546.326563:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574546.326567:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574546.326570:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574546.326573:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574546.326576:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574546.326579:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574546.326582:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330370:330370:256:4294967295] 192.168.202.110@tcp LPNI seq info [330327:330327:8:4294967295]
00000400:00000200:1.0:1788574546.326587:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574546.326595:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574546.326600:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574546.326604:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574546.326608:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574546.326611:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1788574546.326616:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574546.326619:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1788574546.326624:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574546.326626:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574546.326629:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574546.326633:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574546.326636:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574546.326638:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574546.326641:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574546.326645:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326655:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574546.326658:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326666:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574546.326668:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574546.326672:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574546.326675:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574546.326679:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574546.326681:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306149927
00000100:00000001:1.0:1788574546.326684:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306150841
00000100:00000001:1.0:1788574546.326687:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306149927
00000100:00000001:1.0:1788574546.326689:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306150841
00000100:00000001:1.0:1788574546.326692:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574546.326694:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574546.326696:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574546.326700:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574546/real 1788574546]  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574546.326709:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574546.326711:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574546.326713:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574546.326716:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574546.326718:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574546.326720:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574546.326723:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574546.326732:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326739:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574546.326741:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.326743:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574546.326745:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574546.326749:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326756:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574546.326760:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326769:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574546.326772:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.326774:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574546.326775:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.326777:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574546.326780:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574546.326783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574546.326786:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574546.326789:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574546.326793:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574546.326802:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142656:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574546.326807:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574546.326811:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326819:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574546.326820:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574546.326823:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574546.326825:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574546.326827:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574546.326830:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574546.326834:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574546.326836:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574546.326838:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574546.326840:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574546.326844:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759800.
02000000:00000001:1.0:1788574546.326846:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574546.326851:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574546.326854:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574546.326857:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574546.326861:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393142784, portal 4
00000100:00000001:1.0:1788574546.326863:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574546.326864:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:1.0:1788574546.326871:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326879:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574546.326882:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574546.326884:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574546.326887:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393142784, offset 0
00000400:00000200:1.0:1788574546.326891:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574546.326895:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574546.326899:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574546.326903:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574546.326906:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574546.326908:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574546.326911:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574546.326914:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330371:330371:256:4294967295] 192.168.202.110@tcp LPNI seq info [330328:330328:8:4294967295]
00000400:00000200:1.0:1788574546.326919:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574546.326926:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574546.326931:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574546.326935:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574546.326939:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574546.326942:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1788574546.326947:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574546.326950:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1788574546.326954:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574546.326956:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574546.326959:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574546.326962:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574546.326966:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574546.326968:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574546.326971:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574546.326975:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326983:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574546.326986:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.326993:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574546.326995:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574546.326999:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574546.327002:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574546.327006:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574546.327008:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306150073
00000100:00000001:1.0:1788574546.327010:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306150841
00000100:00000001:1.0:1788574546.327013:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306150073
00000100:00000001:1.0:1788574546.327015:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306150841
00000100:00000001:1.0:1788574546.327017:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574546.327020:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574546.327022:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574546.327026:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574546/real 1788574546]  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574546.327035:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574546.327037:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574546.327038:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574546.327041:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574546.327043:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574546.327046:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574546.327049:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574546.327059:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.327072:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574546.327074:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.327076:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574546.327077:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574546.327082:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.327089:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574546.327093:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.327101:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574546.327104:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.327106:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574546.327107:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.327109:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574546.327111:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574546.327114:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574546.327118:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574546.327120:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574546.327125:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574546.327133:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393142784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574546.327137:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.327139:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574546.327141:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393142656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.327146:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574546.327147:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574546.327149:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758000.
02000000:00000001:1.0:1788574546.327151:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574546.327152:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574546.327153:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574546.327154:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574546.327156:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:1.0:1788574546.327158:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574546.327158:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574546.327160:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:1.0:1788574546.327161:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574546.327163:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.327164:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574546.327166:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393142784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574546.327170:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574546.327171:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574546.327173:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759800.
02000000:00000001:1.0:1788574546.327175:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574546.327176:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574546.327178:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574546.327180:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574546.327181:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:1.0:1788574546.327183:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574546.327184:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574546.327186:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:1.0:1788574546.327188:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574546.327190:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.327192:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574546.327196:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574546.327197:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574546.327199:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574546.327201:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574547.285470:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306150841, at 4306150848
00000100:00000040:0.0:1788574547.285472:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306150841, at 4306150848
00000100:00000010:0.0:1788574547.285489:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9240.
00000100:00000010:1.0:1788574547.285492:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9b40.
00000100:00080000:1.0:1788574551.445493:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574551.445502:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574551.445505:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574551.445508:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574551.445512:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574551.445518:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11482
00000400:00000200:1.0:1788574551.445525:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574551.445530:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574551.445532:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574551.445535:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445537:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574551.445539:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445541:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574551.445544:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574551.445545:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574551.445547:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574551.445550:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574551.445552:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574551.445553:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574551.445555:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445556:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574551.445557:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445559:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574551.445561:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574551.445563:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445565:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574551.445567:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574551.445570:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574551.445572:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574551.445573:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574551.445576:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574551.445584:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788574551.445586:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574551.445589:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574551.445590:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574551.445592:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574551.445595:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574551.445598:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574551.445604:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574551.445609:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574551.445616:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574551.445625:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574551.445628:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574551.445631:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574551.445651:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574551.445653:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574551.445656:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574551.445657:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574551.445658:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574551.445661:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574551.445663:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11482
00000400:00000200:1.0:1788574551.445666:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574551.445669:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574551.445670:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1788574551.445672:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788574551.445673:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445676:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788574551.445677:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1788574551.445679:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574551.445681:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1788574551.445682:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574551.445685:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574551.445687:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574551.445688:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:0.0:1788574551.445690:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788574551.445690:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574551.445694:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574551.445698:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574551.445704:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445708:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574551.445711:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445715:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788574551.445716:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1788574551.445717:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788574551.445720:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574551.445721:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445726:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574551.445728:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574551.445728:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574551.445733:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574551.445734:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574551.445737:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574551.445740:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1788574551.445741:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1788574551.445743:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00100000:0.0:1788574551.445748:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393142912:192.168.202.110@tcp:250:kworker.0
00000008:00000001:1.0:1788574551.445748:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574551.445752:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574551.445756:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:1.0:1788574551.445757:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788574551.445761:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574551.445762:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574551.445763:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574551.445767:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574551.445767:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574551.445770:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574551.445771:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1788574551.445773:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574551.445775:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574551.445778:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000010:1.0:1788574551.445779:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
02000000:00000001:0.0:1788574551.445781:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574551.445784:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574551.445788:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:1.0:1788574551.445792:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1788574551.445794:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000100:00000001:1.0:1788574551.445801:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1788574551.445802:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000001:1.0:1788574551.445803:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574551.445808:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574551.445809:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393142912, portal 25
00000100:00000001:0.0:1788574551.445814:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574551.445818:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00000040:0.0:1788574551.445827:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574576 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574551.445828:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574551.445832:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574551.445839:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574551.445841:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574551.445845:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574551.445846:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1788574551.445850:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:0.0:1788574551.445853:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788574551.445856:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11482
00000400:00000010:0.0:1788574551.445857:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000400:00000200:1.0:1788574551.445862:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:0.0:1788574551.445863:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393142912, offset 0
00000400:00000200:1.0:1788574551.445866:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574551.445869:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1788574551.445873:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788574551.445873:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445877:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574551.445880:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445882:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574551.445883:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574551.445884:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574551.445885:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574551.445886:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574551.445887:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574551.445889:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788574551.445894:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788574551.445894:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445900:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788574551.445902:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:1.0:1788574551.445905:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574551.445910:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1788574551.445910:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788574551.445916:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:1.0:1788574551.445916:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:0.0:1788574551.445922:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:1.0:1788574551.445924:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1788574551.445927:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:1.0:1788574551.445931:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574551.445933:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330372:330372:256:4294967295] 192.168.202.110@tcp LPNI seq info [330329:330329:8:4294967295]
02000000:00000001:1.0:1788574551.445937:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1788574551.445939:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788574551.445943:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574551.445949:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574551.445953:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1788574551.445953:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574551.445957:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574551.445960:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574551.445963:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788574551.445964:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000010:1.0:1788574551.445966:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000800:00000010:0.0:1788574551.445970:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000020:00000040:1.0:1788574551.445970:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574551.445975:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574551.445976:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574551.445978:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574551.445980:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:0.0:1788574551.445981:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1788574551.445982:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788574551.445987:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
02000000:00000010:1.0:1788574551.445989:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574551.445994:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788574551.445995:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:1.0:1788574551.445999:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1788574551.446006:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1788574551.446010:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574551.446014:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1788574551.446015:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574551.446018:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000100:00000040:1.0:1788574551.446019:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788574551.446024:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1788574551.446026:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574551.446030:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788574551.446031:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:1.0:1788574551.446037:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:0.0:1788574551.446039:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574551.446042:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000040:1.0:1788574551.446043:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11492)
00000100:00000001:0.0:1788574551.446048:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574551.446053:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574576 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446061:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574551.446064:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574576 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446069:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574551.446070:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574551.446075:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574551.446077:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574551.446079:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306155961
00000100:00000010:0.0:1788574551.446084:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574551.446090:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574551.446095:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306155961
00000100:00000010:0.0:1788574551.446097:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ae0.
00000100:00000001:0.0:1788574551.446099:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574551.446101:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574551.446103:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574551.446105:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574551.446108:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574551/real 1788574551]  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574551.446115:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574551.446116:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574551.446118:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574551.446125:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574551.446126:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574551.446128:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574551.446130:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574551.446136:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446141:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574551.446142:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446143:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574551.446145:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574551.446147:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446152:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574551.446155:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446160:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574551.446162:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446163:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574551.446164:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446166:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574551.446167:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574551.446169:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574551.446171:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574551.446173:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574551.446175:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574551.446180:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393142912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574551.446184:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446185:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574551.446187:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393142912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446192:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574551.446193:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574551.446196:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574551.446198:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574551.446199:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574551.446200:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574551.446201:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574551.446203:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:0.0:1788574551.446206:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574551.446208:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574551.446209:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:0.0:1788574551.446211:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574551.446212:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446214:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446219:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574551.446221:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574551.446222:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574551.446223:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574551.446224:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574551.446226:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446231:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574551.446232:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574551.446234:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574551.446235:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574551.446237:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574551.446238:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574551.446240:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574551.446242:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574551.446243:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574551.446244:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574551.446246:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574551.446247:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574551.446249:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574551.446251:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574551.446253:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574551.446255:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393143040, portal 10
00000100:00000001:0.0:1788574551.446257:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574551.446258:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:0.0:1788574551.446263:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574576 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446267:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574551.446269:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574551.446271:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574551.446272:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393143040, offset 0
00000400:00000200:0.0:1788574551.446275:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574551.446277:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574551.446280:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574551.446282:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574551.446284:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574551.446285:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574551.446287:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574551.446289:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330373:330373:256:4294967295] 192.168.202.110@tcp LPNI seq info [330330:330330:8:4294967295]
00000400:00000200:0.0:1788574551.446292:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574551.446296:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574551.446298:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574551.446300:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574551.446303:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574551.446305:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788574551.446308:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574551.446309:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788574551.446312:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574551.446313:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574551.446315:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574551.446318:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574551.446320:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574551.446321:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574551.446322:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574551.446325:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574576 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446337:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574551.446339:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574576 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446344:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574551.446345:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574551.446348:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574551.446350:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574551.446353:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574551.446354:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306155046
00000100:00000001:0.0:1788574551.446356:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306155961
00000100:00000001:0.0:1788574551.446357:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306155046
00000100:00000001:0.0:1788574551.446359:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306155961
00000100:00000001:0.0:1788574551.446360:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574551.446362:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574551.446363:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574551.446365:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574551/real 1788574551]  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574551.446370:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574551.446371:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574551.446372:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574551.446374:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574551.446375:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574551.446376:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574551.446378:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574551.446383:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446387:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574551.446388:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446389:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574551.446422:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574551.446426:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446430:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574551.446433:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446438:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574551.446440:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446441:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574551.446442:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446443:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574551.446444:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574551.446446:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574551.446448:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574551.446449:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574551.446452:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574551.446457:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574551.446460:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574551.446462:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446466:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574551.446467:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574551.446468:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574551.446470:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574551.446471:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574551.446473:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143168:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574551.446475:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574551.446476:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574551.446478:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574551.446479:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574551.446481:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574551.446483:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574551.446486:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574551.446487:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574551.446490:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574551.446492:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393143168, portal 4
00000100:00000001:0.0:1788574551.446494:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574551.446494:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000040:0.0:1788574551.446499:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574576 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446503:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574551.446505:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574551.446506:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574551.446508:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393143168, offset 0
00000400:00000200:0.0:1788574551.446510:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574551.446513:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574551.446515:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574551.446517:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574551.446519:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574551.446521:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574551.446522:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574551.446525:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330374:330374:256:4294967295] 192.168.202.110@tcp LPNI seq info [330331:330331:8:4294967295]
00000400:00000200:0.0:1788574551.446533:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574551.446538:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574551.446541:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574551.446544:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574551.446546:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574551.446548:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788574551.446551:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574551.446553:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788574551.446555:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574551.446557:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574551.446559:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574551.446561:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574551.446564:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574551.446565:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574551.446567:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574551.446569:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574576 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446575:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574551.446577:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574576 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446581:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574551.446582:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574551.446584:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574551.446586:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574551.446588:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574551.446589:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306155193
00000100:00000001:0.0:1788574551.446591:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306155961
00000100:00000001:0.0:1788574551.446592:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306155193
00000100:00000001:0.0:1788574551.446594:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306155961
00000100:00000001:0.0:1788574551.446595:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574551.446596:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574551.446597:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574551.446599:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574551/real 1788574551]  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574551.446605:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574551.446606:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574551.446607:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574551.446608:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574551.446609:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574551.446610:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574551.446612:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574551.446617:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446626:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574551.446627:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574551.446629:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574551.446631:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446635:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574551.446637:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446641:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574551.446643:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446644:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574551.446645:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446646:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574551.446647:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574551.446648:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574551.446650:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574551.446652:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574551.446654:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574551.446659:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143168:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574551.446661:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446663:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574551.446665:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393143040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446669:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574551.446670:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574551.446671:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574551.446673:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574551.446674:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574551.446675:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574551.446676:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574551.446677:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:0.0:1788574551.446679:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574551.446680:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574551.446681:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:0.0:1788574551.446683:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574551.446683:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446684:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574551.446686:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393143168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574576 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574551.446692:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574551.446693:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574551.446695:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574551.446697:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574551.446699:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574551.446700:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574551.446702:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574551.446703:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:0.0:1788574551.446706:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574551.446707:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574551.446708:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:0.0:1788574551.446710:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574551.446712:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446714:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574551.446718:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574551.446719:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574551.446721:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574551.446722:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574552.405480:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306155961, at 4306155968
00000100:00000040:0.0:1788574552.405491:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306155961, at 4306155968
00000100:00000010:1.0:1788574552.405498:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ae0.
00000100:00000010:0.0:1788574552.405500:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00080000:1.0:1788574556.565471:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574556.565535:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574556.565538:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574556.565541:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574556.565545:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574556.565552:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11487
00000400:00000200:1.0:1788574556.565561:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574556.565567:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574556.565571:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574556.565574:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565578:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574556.565581:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565584:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574556.565588:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574556.565590:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574556.565593:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574556.565596:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574556.565598:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574556.565600:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574556.565602:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565604:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574556.565605:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565607:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574556.565609:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574556.565611:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565613:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574556.565615:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574556.565618:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574556.565620:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574556.565621:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574556.565623:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574556.565631:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788574556.565634:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574556.565651:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574556.565652:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574556.565655:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574556.565657:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574556.565661:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574556.565667:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788574556.565673:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574556.565680:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574556.565688:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574556.565692:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574556.565695:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574556.565714:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574556.565716:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574556.565719:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574556.565720:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574556.565721:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574556.565724:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574556.565726:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11487
00000400:00000200:1.0:1788574556.565730:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574556.565733:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574556.565735:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1788574556.565737:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788574556.565738:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565740:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574556.565741:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565743:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574556.565744:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574556.565745:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574556.565746:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574556.565747:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574556.565749:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574556.565751:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574556.565753:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565754:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574556.565755:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565757:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574556.565758:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574556.565762:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574556.565763:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574556.565765:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574556.565767:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788574556.565770:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1788574556.565772:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574556.565775:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574556.565776:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:0.0:1788574556.565780:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:1.0:1788574556.565781:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574556.565786:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574556.565790:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574556.565795:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000100:00000001:0.0:1788574556.565797:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1788574556.565798:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:0.0:1788574556.565800:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574556.565801:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574556.565804:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574556.565804:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574556.565806:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574556.565808:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574556.565808:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574556.565812:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574556.565813:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574556.565817:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393143296:192.168.202.110@tcp:250:kworker.0
02000000:00000010:1.0:1788574556.565817:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:0.0:1788574556.565824:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1788574556.565825:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574556.565827:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574556.565830:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574556.565832:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:1.0:1788574556.565836:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:0.0:1788574556.565839:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574556.565843:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574556.565852:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574556.565853:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1788574556.565856:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000100:00000001:1.0:1788574556.565857:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574556.565864:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000040:1.0:1788574556.565866:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574556.565871:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393143296, portal 25
00000100:00000001:0.0:1788574556.565876:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574556.565879:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
00000100:00000040:0.0:1788574556.565888:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574556.565889:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574556.565894:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574556.565902:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574556.565905:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1788574556.565906:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:1.0:1788574556.565909:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:0.0:1788574556.565912:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565914:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:0.0:1788574556.565916:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00080000:1.0:1788574556.565919:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11487
00000100:00000200:0.0:1788574556.565920:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393143296, offset 0
00000400:00000200:1.0:1788574556.565925:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574556.565930:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574556.565932:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574556.565935:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574556.565937:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565938:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788574556.565940:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788574556.565943:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565946:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574556.565948:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1788574556.565950:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:1.0:1788574556.565950:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574556.565952:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574556.565955:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:0.0:1788574556.565957:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:1.0:1788574556.565958:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574556.565961:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788574556.565962:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:1.0:1788574556.565966:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574556.565969:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1788574556.565970:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574556.565983:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574556.565984:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:1.0:1788574556.565988:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788574556.565992:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330375:330375:256:4294967295] 192.168.202.110@tcp LPNI seq info [330332:330332:8:4294967295]
00000020:00000040:1.0:1788574556.565993:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:0.0:1788574556.566001:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788574556.566003:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574556.566007:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574556.566013:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1788574556.566014:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1788574556.566019:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.0:1788574556.566021:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788574556.566026:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788574556.566029:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000100:00000001:1.0:1788574556.566030:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574556.566035:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:0.0:1788574556.566036:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000008:00000001:1.0:1788574556.566040:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788574556.566042:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:1.0:1788574556.566043:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574556.566051:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000800:00000200:0.1:1788574556.566052:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:1.0:1788574556.566056:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:0.1:1788574556.566057:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:1.0:1788574556.566061:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000100:0.0:1788574556.566064:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788574556.566065:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788574556.566067:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000100:00000001:1.0:1788574556.566071:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574556.566074:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1788574556.566075:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:1.0:1788574556.566078:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574556.566081:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574556.566087:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
02000000:00000010:1.0:1788574556.566087:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000400:00000010:0.0:1788574556.566089:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:1.0:1788574556.566094:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566095:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574556.566100:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574556.566106:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574556.566109:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574556.566116:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574581 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574556.566121:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574556.566124:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566126:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566129:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:1.0:1788574556.566131:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566134:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574556.566141:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574556.566143:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574556.566147:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:0.0:1788574556.566148:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574556.566152:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306161081
00000100:00000040:1.0:1788574556.566152:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11497)
00000100:00000010:0.0:1788574556.566159:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574556.566167:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574556.566172:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306161081
00000100:00000010:0.0:1788574556.566174:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d79c0.
00000100:00000001:0.0:1788574556.566177:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574556.566179:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574556.566181:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566183:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574556.566187:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574556/real 1788574556]  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574556.566194:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574556.566195:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574556.566196:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574556.566198:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574556.566200:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574556.566201:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574556.566204:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574556.566209:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566214:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574556.566215:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566217:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574556.566218:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574556.566221:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566225:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574556.566228:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566233:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574556.566235:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566237:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574556.566238:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566239:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574556.566241:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574556.566243:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574556.566246:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574556.566248:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574556.566250:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574556.566256:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393143296:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574556.566260:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566261:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574556.566263:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393143296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566267:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574556.566269:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574556.566271:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574556.566273:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574556.566274:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574556.566275:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574556.566277:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574556.566279:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:0.0:1788574556.566281:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574556.566283:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574556.566284:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:0.0:1788574556.566286:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574556.566286:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566288:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566293:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574556.566295:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566296:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574556.566297:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574556.566299:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574556.566301:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566306:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574556.566307:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574556.566309:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574556.566310:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574556.566311:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574556.566313:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143424:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574556.566315:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574556.566317:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574556.566318:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574556.566319:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574556.566321:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574556.566322:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574556.566324:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574556.566326:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574556.566328:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574556.566330:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393143424, portal 10
00000100:00000001:0.0:1788574556.566332:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574556.566333:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345473920 : -130483364077696 : ffff8953796f5f80)
00000100:00000040:0.0:1788574556.566337:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566342:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574556.566344:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574556.566345:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574556.566347:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393143424, offset 0
00000400:00000200:0.0:1788574556.566349:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574556.566351:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574556.566354:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574556.566356:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574556.566358:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574556.566359:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574556.566361:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574556.566363:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330376:330376:256:4294967295] 192.168.202.110@tcp LPNI seq info [330333:330333:8:4294967295]
00000400:00000200:0.0:1788574556.566366:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574556.566370:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574556.566374:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574556.566376:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574556.566378:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574556.566380:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574556.566384:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574556.566385:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574556.566388:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574556.566389:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574556.566408:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574556.566411:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574556.566426:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574556.566427:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574556.566429:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574556.566433:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566438:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574556.566440:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574581 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566444:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566446:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574556.566448:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574556.566451:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574556.566453:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574556.566455:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306160167
00000100:00000001:0.0:1788574556.566457:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306161081
00000100:00000001:0.0:1788574556.566459:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306160167
00000100:00000001:0.0:1788574556.566461:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306161081
00000100:00000001:0.0:1788574556.566462:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574556.566464:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566465:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574556.566467:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574556/real 1788574556]  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574556.566474:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574556.566474:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574556.566476:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574556.566477:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574556.566478:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574556.566479:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574556.566481:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574556.566487:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566491:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574556.566492:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566493:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574556.566494:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574556.566496:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566500:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574556.566502:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566509:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574556.566512:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566513:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574556.566515:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566516:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574556.566518:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574556.566521:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574556.566525:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574556.566527:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574556.566532:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574556.566541:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143424:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574556.566546:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574556.566549:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566556:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574556.566558:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574556.566560:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574556.566561:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574556.566563:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574556.566565:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143552:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574556.566567:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574556.566569:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574556.566570:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574556.566571:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574556.566574:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574556.566575:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574556.566578:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574556.566580:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574556.566582:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574556.566584:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393143552, portal 4
00000100:00000001:0.0:1788574556.566586:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574556.566587:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345465856 : -130483364085760 : ffff8953796f4000)
00000100:00000040:0.0:1788574556.566591:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566595:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574556.566597:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574556.566598:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574556.566600:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393143552, offset 0
00000400:00000200:0.0:1788574556.566603:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574556.566606:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574556.566608:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574556.566610:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574556.566612:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574556.566614:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574556.566615:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574556.566618:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330377:330377:256:4294967295] 192.168.202.110@tcp LPNI seq info [330334:330334:8:4294967295]
00000400:00000200:0.0:1788574556.566620:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574556.566624:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574556.566628:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574556.566630:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574556.566632:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574556.566634:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574556.566637:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574556.566639:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574556.566641:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574556.566643:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574556.566645:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574556.566647:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574556.566649:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574556.566651:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574556.566652:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574556.566654:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566660:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574556.566662:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574581 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566666:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566667:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574556.566669:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574556.566671:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574556.566673:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574556.566675:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306160313
00000100:00000001:0.0:1788574556.566676:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306161081
00000100:00000001:0.0:1788574556.566678:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306160313
00000100:00000001:0.0:1788574556.566679:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306161081
00000100:00000001:0.0:1788574556.566680:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574556.566682:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566683:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574556.566685:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574556/real 1788574556]  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574556.566690:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574556.566691:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574556.566692:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574556.566694:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574556.566695:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574556.566696:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574556.566698:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574556.566704:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566708:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574556.566709:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566710:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574556.566710:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574556.566713:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566716:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574556.566719:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566723:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574556.566725:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566726:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574556.566727:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566728:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574556.566729:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574556.566731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574556.566733:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574556.566734:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574556.566737:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574556.566741:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143552:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574556.566744:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566745:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574556.566747:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5f80 x1875444393143424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566751:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574556.566752:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574556.566753:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574556.566754:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574556.566755:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574556.566756:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574556.566757:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574556.566759:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:0.0:1788574556.566761:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574556.566761:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574556.566762:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5f80.
00000100:00000001:0.0:1788574556.566764:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574556.566765:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566766:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574556.566768:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4000 x1875444393143552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574581 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574556.566777:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574556.566777:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574556.566778:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574556.566780:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574556.566781:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574556.566781:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574556.566782:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574556.566783:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:0.0:1788574556.566785:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574556.566785:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574556.566786:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4000.
00000100:00000001:0.0:1788574556.566787:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574556.566788:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566789:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574556.566791:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574556.566792:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574556.566793:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574556.566795:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574557.525520:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306161081, at 4306161088
00000100:00000040:1.0:1788574557.525520:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306161081, at 4306161088
00000100:00000010:0.0:1788574557.525551:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574557.525558:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d79c0.
00000800:00000200:0.0:1788574559.637482:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788574559.637488:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788574559.637492:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788574559.637631:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1788574559.637641:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788574559.637645:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788574559.637650:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574559.637653:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.686249:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574561.686260:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574561.686267:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574561.686284:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.686305:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574561.686311:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574561.686320:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574561.686325:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574561.686328:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574561.686333:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393143680:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574561.686354:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574561.686365:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574561.686371:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574561.686377:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574561.686425:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975a800.
02000000:00000001:1.0:1788574561.686429:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574561.686438:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574561.686443:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574561.686452:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574561.686467:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393143680, portal 25
00000100:00000001:1.0:1788574561.686472:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574561.686476:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:1.0:1788574561.686488:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.686514:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574561.686534:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574561.686541:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574561.686546:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393143680, offset 0
00000400:00000200:1.0:1788574561.686559:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574561.686574:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574561.686590:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574561.686605:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574561.686616:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574561.686628:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574561.686636:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574561.686648:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330378:330378:256:4294967295] 192.168.202.110@tcp LPNI seq info [330335:330335:8:4294967295]
00000400:00000200:1.0:1788574561.686661:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574561.686679:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574561.686694:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574561.686706:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574561.686724:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574561.686735:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1788574561.686759:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574561.686766:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1788574561.686777:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574561.686782:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574561.686794:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574561.686804:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574561.686814:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574561.686818:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574561.686826:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574561.686832:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.686853:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574561.686859:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.686868:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574561.686870:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574561.686880:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574561.686886:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574561.686890:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306166202
00000100:00000010:1.0:1788574561.686898:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9b40.
00000100:00000001:1.0:1788574561.686911:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574561.686925:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306166202
00000100:00000010:1.0:1788574561.686928:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9960.
00000100:00000001:1.0:1788574561.686931:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574561.686934:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574561.686937:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574561.686940:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574561.686945:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574561/real 1788574561]  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574561.686955:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574561.686965:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574561.686968:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574561.686973:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574561.686975:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574561.686979:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574561.686983:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574561.686993:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687001:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574561.687004:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687007:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574561.687010:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574561.687015:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687025:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574561.687030:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687043:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574561.687046:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687048:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574561.687049:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687052:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574561.687054:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574561.687056:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574561.687061:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574561.687065:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574561.687071:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574561.687080:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393143680:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574561.687089:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687092:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574561.687097:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393143680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687108:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574561.687112:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574561.687115:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975a800.
02000000:00000001:1.0:1788574561.687120:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574561.687122:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574561.687123:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574561.687126:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574561.687130:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:1.0:1788574561.687134:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574561.687137:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574561.687139:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:1.0:1788574561.687143:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574561.687144:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687146:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687155:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574561.687157:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574561.687159:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574561.687161:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574561.687162:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574561.687165:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687171:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574561.687172:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574561.687174:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574561.687175:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574561.687177:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574561.687180:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574561.687182:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574561.687184:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574561.687185:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574561.687187:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574561.687190:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975a800.
02000000:00000001:1.0:1788574561.687193:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574561.687197:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574561.687200:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574561.687205:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574561.687209:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393143808, portal 10
00000100:00000001:1.0:1788574561.687211:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574561.687213:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:1.0:1788574561.687221:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687229:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574561.687231:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574561.687234:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574561.687238:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393143808, offset 0
00000400:00000200:1.0:1788574561.687243:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574561.687248:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574561.687252:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574561.687255:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574561.687258:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574561.687261:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574561.687264:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574561.687267:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330379:330379:256:4294967295] 192.168.202.110@tcp LPNI seq info [330336:330336:8:4294967295]
00000400:00000200:1.0:1788574561.687272:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574561.687279:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574561.687284:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574561.687288:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574561.687293:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574561.687297:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1788574561.687301:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574561.687304:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1788574561.687309:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574561.687311:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574561.687315:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574561.687319:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574561.687323:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574561.687325:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574561.687328:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574561.687332:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687341:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574561.687344:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687351:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574561.687353:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574561.687357:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574561.687362:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574561.687365:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574561.687367:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306165287
00000100:00000001:1.0:1788574561.687372:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306166202
00000100:00000001:1.0:1788574561.687374:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306165287
00000100:00000001:1.0:1788574561.687377:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306166202
00000100:00000001:1.0:1788574561.687380:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574561.687382:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574561.687422:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574561.687428:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574561/real 1788574561]  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574561.687444:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574561.687446:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574561.687449:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574561.687452:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574561.687454:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574561.687456:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574561.687460:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574561.687469:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687476:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574561.687478:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687479:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574561.687480:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574561.687484:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687492:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574561.687497:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687505:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574561.687508:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687510:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574561.687511:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687514:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574561.687516:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574561.687519:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574561.687523:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574561.687526:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574561.687531:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574561.687540:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574561.687546:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574561.687550:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687558:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574561.687560:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574561.687563:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574561.687565:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574561.687567:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574561.687571:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574561.687575:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574561.687577:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574561.687578:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574561.687580:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574561.687584:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574561.687587:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574561.687592:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574561.687596:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000010:1.0:1788574561.687599:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000100:00000200:1.0:1788574561.687602:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393143936, portal 4
00000100:00000001:1.0:1788574561.687605:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574561.687606:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:1.0:1788574561.687615:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687622:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574561.687625:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574561.687628:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574561.687630:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393143936, offset 0
00000400:00000200:1.0:1788574561.687636:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574561.687640:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574561.687644:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574561.687647:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574561.687650:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574561.687652:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574561.687655:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574561.687658:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330380:330380:256:4294967295] 192.168.202.110@tcp LPNI seq info [330337:330337:8:4294967295]
00000400:00000200:1.0:1788574561.687664:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574561.687671:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574561.687676:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574561.687681:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574561.687686:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574561.687689:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1788574561.687694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574561.687697:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1788574561.687701:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574561.687703:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574561.687706:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574561.687710:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574561.687714:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574561.687715:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574561.687718:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574561.687722:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574561.687734:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687741:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574561.687743:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574561.687746:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574561.687751:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574561.687755:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574561.687758:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306165434
00000100:00000001:1.0:1788574561.687761:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306166202
00000100:00000001:1.0:1788574561.687764:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306165434
00000100:00000001:1.0:1788574561.687767:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306166202
00000100:00000001:1.0:1788574561.687769:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574561.687771:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574561.687773:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574561.687778:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574561/real 1788574561]  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574561.687786:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574561.687788:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574561.687789:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:1.0:1788574561.687791:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:1.0:1788574561.687793:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:1.0:1788574561.687796:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574561.687800:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574561.687809:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687817:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574561.687819:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687822:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574561.687824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574561.687828:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687835:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574561.687840:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687847:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574561.687851:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687852:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574561.687854:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687856:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574561.687859:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574561.687862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574561.687866:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574561.687868:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574561.687872:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574561.687881:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393143936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574561.687886:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687888:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574561.687892:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393143808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687898:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574561.687900:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574561.687902:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975a800.
02000000:00000001:1.0:1788574561.687905:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574561.687907:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574561.687909:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574561.687912:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574561.687914:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:1.0:1788574561.687917:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574561.687919:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574561.687921:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:1.0:1788574561.687925:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574561.687926:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687928:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574561.687932:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393143936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574561.687946:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574561.687948:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574561.687949:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574561.687952:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574561.687953:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574561.687954:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574561.687957:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574561.687959:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:1.0:1788574561.687961:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574561.687962:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574561.687964:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:1.0:1788574561.687966:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574561.687967:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687970:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574561.687975:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574561.687976:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574561.687978:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574561.687980:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574562.645505:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306166202, at 4306166208
00000100:00000040:1.0:1788574562.645516:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306166202, at 4306166208
00000100:00000010:0.0:1788574562.645523:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9b40.
00000100:00000010:1.0:1788574562.645532:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9960.
00000100:00080000:1.0:1788574566.805509:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574566.805518:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574566.805520:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574566.805523:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574566.805528:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574566.805534:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11497
00000400:00000200:1.0:1788574566.805543:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574566.805548:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574566.805550:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574566.805553:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805555:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574566.805557:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805560:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574566.805563:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574566.805564:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574566.805566:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574566.805569:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574566.805571:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574566.805572:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574566.805574:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805575:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574566.805576:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805577:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574566.805580:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574566.805582:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805584:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574566.805587:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574566.805590:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574566.805592:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574566.805594:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574566.805596:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574566.805601:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788574566.805603:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574566.805606:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574566.805607:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574566.805609:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574566.805611:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574566.805614:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574566.805620:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788574566.805625:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574566.805632:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574566.805640:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574566.805644:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574566.805648:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574566.805663:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574566.805666:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574566.805668:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574566.805670:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574566.805671:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574566.805673:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574566.805674:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11497
00000400:00000200:1.0:1788574566.805676:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574566.805678:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574566.805680:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574566.805682:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805684:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574566.805687:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805688:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574566.805690:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574566.805691:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574566.805692:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574566.805694:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574566.805695:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788574566.805696:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1788574566.805697:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788574566.805701:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1788574566.805701:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805705:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788574566.805707:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574566.805707:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805712:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574566.805714:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574566.805718:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:0.0:1788574566.805720:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574566.805720:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574566.805724:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574566.805729:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574566.805733:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574566.805736:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574566.805741:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574566.805746:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574566.805749:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574566.805754:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000100:00000001:0.0:1788574566.805756:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1788574566.805757:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:0.0:1788574566.805759:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574566.805761:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574566.805763:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574566.805764:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788574566.805765:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574566.805768:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574566.805769:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574566.805770:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574566.805774:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574566.805776:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574566.805780:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574566.805781:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393144064:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574566.805787:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:1.0:1788574566.805788:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1788574566.805792:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574566.805795:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574566.805800:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788574566.805800:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574566.805803:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574566.805807:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
00000100:00000040:1.0:1788574566.805807:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788574566.805812:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574566.805821:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574566.805823:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574566.805825:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000100:00080000:1.0:1788574566.805828:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1788574566.805832:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00080000:1.0:1788574566.805833:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574566.805838:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574566.805840:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:0.0:1788574566.805842:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393144064, portal 25
00000100:00000001:1.0:1788574566.805844:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:0.0:1788574566.805846:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574566.805851:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345466752 : -130483364084864 : ffff8953796f4380)
00000100:00080000:1.0:1788574566.805851:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11497
00000400:00000200:1.0:1788574566.805858:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788574566.805864:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574566.805865:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574566.805871:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574566.805875:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805877:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574566.805881:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805886:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574566.805892:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574566.805893:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:0.0:1788574566.805898:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:1.0:1788574566.805898:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574566.805902:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000040:0.0:1788574566.805904:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:1.0:1788574566.805908:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574566.805912:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574566.805916:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805918:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574566.805920:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805922:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574566.805925:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:0.0:1788574566.805928:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00080000:1.0:1788574566.805930:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574566.805935:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574566.805937:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393144064, offset 0
02000000:00000001:1.0:1788574566.805938:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574566.805942:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574566.805946:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574566.805950:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1788574566.805951:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00100000:1.0:1788574566.805953:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574566.805957:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574566.805959:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574566.805964:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:1.0:1788574566.805965:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788574566.805969:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574566.805971:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574566.805974:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574566.805975:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:1.0:1788574566.805976:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574566.805980:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574566.805983:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574566.805984:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000010:1.0:1788574566.805990:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000400:00000200:0.0:1788574566.805991:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:1.0:1788574566.805996:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574566.805997:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574566.806003:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00100000:1.0:1788574566.806006:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788574566.806013:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330381:330381:256:4294967295] 192.168.202.110@tcp LPNI seq info [330338:330338:8:4294967295]
00000100:00000001:1.0:1788574566.806016:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574566.806020:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574566.806026:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788574566.806029:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574566.806037:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574566.806040:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574566.806046:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:0.0:1788574566.806051:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1788574566.806051:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11508)
00000800:00000200:0.0:1788574566.806062:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574566.806069:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574566.806086:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574566.806090:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788574566.806095:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574566.806098:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1788574566.806102:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574566.806104:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574566.806108:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574566.806112:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574566.806116:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574566.806118:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574566.806121:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574566.806125:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806135:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574566.806139:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574591 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806147:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574566.806148:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574566.806155:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574566.806160:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574566.806164:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306171321
00000100:00000010:0.0:1788574566.806169:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574566.806176:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574566.806182:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306171321
00000100:00000010:0.0:1788574566.806184:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7360.
00000100:00000001:0.0:1788574566.806187:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574566.806191:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574566.806194:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574566.806197:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574566.806202:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574566/real 1788574566]  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574566.806213:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574566.806215:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574566.806218:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574566.806221:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574566.806223:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574566.806226:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574566.806230:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574566.806240:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806247:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574566.806249:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806252:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574566.806254:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574566.806259:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806267:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574566.806272:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806279:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574566.806281:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806283:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574566.806283:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806285:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574566.806287:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574566.806289:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574566.806291:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574566.806294:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574566.806297:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574566.806302:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393144064:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574566.806306:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806307:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574566.806309:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4380 x1875444393144064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806313:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574566.806315:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574566.806317:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574566.806320:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574566.806321:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574566.806322:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574566.806324:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574566.806326:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:0.0:1788574566.806329:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574566.806330:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574566.806331:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4380.
00000100:00000001:0.0:1788574566.806333:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574566.806334:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806336:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806342:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574566.806343:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574566.806344:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574566.806346:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574566.806347:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574566.806349:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806371:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574566.806372:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574566.806374:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574566.806375:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574566.806377:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574566.806380:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144192:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574566.806382:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574566.806384:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574566.806385:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574566.806386:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574566.806389:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574566.806414:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574566.806418:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574566.806421:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574566.806423:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574566.806426:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393144192, portal 10
00000100:00000001:0.0:1788574566.806427:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574566.806428:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345468544 : -130483364083072 : ffff8953796f4a80)
00000100:00000040:0.0:1788574566.806433:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806438:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574566.806440:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574566.806441:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574566.806443:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393144192, offset 0
00000400:00000200:0.0:1788574566.806445:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574566.806448:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574566.806451:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574566.806453:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574566.806455:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574566.806457:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574566.806459:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574566.806461:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330382:330382:256:4294967295] 192.168.202.110@tcp LPNI seq info [330339:330339:8:4294967295]
00000400:00000200:0.0:1788574566.806463:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574566.806468:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574566.806471:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574566.806473:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574566.806476:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574566.806478:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788574566.806481:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574566.806483:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1788574566.806486:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574566.806487:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574566.806489:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574566.806492:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574566.806494:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574566.806496:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574566.806497:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574566.806500:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806514:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574566.806516:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574591 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806521:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574566.806522:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574566.806524:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574566.806526:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574566.806529:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574566.806530:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306170407
00000100:00000001:0.0:1788574566.806532:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306171321
00000100:00000001:0.0:1788574566.806534:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306170407
00000100:00000001:0.0:1788574566.806535:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306171321
00000100:00000001:0.0:1788574566.806537:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574566.806538:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574566.806540:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574566.806542:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574566/real 1788574566]  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574566.806552:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574566.806553:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574566.806554:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574566.806556:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574566.806557:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574566.806558:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574566.806560:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574566.806566:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806574:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574566.806575:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806576:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574566.806577:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574566.806580:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806584:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574566.806587:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806591:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574566.806593:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806594:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574566.806594:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806596:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574566.806597:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574566.806599:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574566.806601:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574566.806602:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574566.806605:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574566.806610:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144192:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574566.806613:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574566.806615:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806619:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574566.806619:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574566.806621:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574566.806622:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574566.806623:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574566.806625:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144320:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574566.806627:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574566.806629:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574566.806630:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574566.806631:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574566.806634:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574566.806635:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574566.806637:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574566.806639:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574566.806641:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574566.806643:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393144320, portal 4
00000100:00000001:0.0:1788574566.806644:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574566.806645:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345476608 : -130483364075008 : ffff8953796f6a00)
00000100:00000040:0.0:1788574566.806650:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806654:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574566.806655:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574566.806657:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574566.806658:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393144320, offset 0
00000400:00000200:0.0:1788574566.806661:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574566.806664:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574566.806666:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574566.806668:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574566.806670:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574566.806671:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574566.806673:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574566.806675:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330383:330383:256:4294967295] 192.168.202.110@tcp LPNI seq info [330340:330340:8:4294967295]
00000400:00000200:0.0:1788574566.806678:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574566.806682:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574566.806685:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574566.806687:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574566.806689:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574566.806691:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788574566.806694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574566.806696:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1788574566.806698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574566.806700:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574566.806702:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574566.806705:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574566.806708:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574566.806710:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574566.806713:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574566.806734:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806743:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574566.806747:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574591 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806755:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574566.806756:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574566.806760:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574566.806764:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574566.806766:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574566.806768:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306170553
00000100:00000001:0.0:1788574566.806770:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306171321
00000100:00000001:0.0:1788574566.806772:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306170553
00000100:00000001:0.0:1788574566.806773:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306171321
00000100:00000001:0.0:1788574566.806774:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574566.806776:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574566.806777:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574566.806780:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574566/real 1788574566]  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574566.806785:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574566.806786:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574566.806788:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574566.806789:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574566.806790:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574566.806792:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574566.806794:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574566.806799:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806804:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574566.806805:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806806:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574566.806807:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574566.806809:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806813:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574566.806815:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806819:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574566.806822:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806823:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574566.806824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806825:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574566.806826:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574566.806828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574566.806830:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574566.806831:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574566.806833:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574566.806839:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144320:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574566.806842:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806844:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574566.806847:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4a80 x1875444393144192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806854:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574566.806855:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574566.806857:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574566.806860:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574566.806861:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574566.806863:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574566.806865:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574566.806867:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:0.0:1788574566.806869:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574566.806871:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574566.806872:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4a80.
00000100:00000001:0.0:1788574566.806875:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574566.806876:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806878:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574566.806882:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6a00 x1875444393144320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574566.806894:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574566.806895:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574566.806896:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574566.806898:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574566.806899:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574566.806900:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574566.806901:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574566.806902:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:0.0:1788574566.806903:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574566.806904:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574566.806905:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6a00.
00000100:00000001:0.0:1788574566.806907:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574566.806908:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806909:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574566.806912:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574566.806913:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574566.806914:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574566.806915:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574567.765506:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306171321, at 4306171328
00000100:00000040:1.0:1788574567.765506:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306171321, at 4306171328
00000100:00000010:0.0:1788574567.765524:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574567.765527:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7360.
00000100:00000001:0.0:1788574571.925699:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574571.925706:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574571.925711:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574571.925723:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.925743:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574571.925747:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574571.925754:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574571.925758:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574571.925766:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574571.925775:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393144448:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574571.925784:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574571.925790:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574571.925796:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574571.925800:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574571.925808:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574571.925814:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574571.925824:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574571.925831:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574571.925841:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574571.925862:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393144448, portal 25
00000100:00000001:0.0:1788574571.925868:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574571.925874:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:0.0:1788574571.925887:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.925909:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574571.925918:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574571.925925:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574571.925930:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393144448, offset 0
00000400:00000200:0.0:1788574571.925942:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574571.925953:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574571.925965:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574571.925975:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574571.925982:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574571.925990:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574571.925997:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574571.926004:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330384:330384:256:4294967295] 192.168.202.110@tcp LPNI seq info [330341:330341:8:4294967295]
00000400:00000200:0.0:1788574571.926017:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574571.926038:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574571.926056:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574571.926065:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574571.926074:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574571.926078:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574571.926085:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574571.926089:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574571.926094:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574571.926097:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574571.926105:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574571.926110:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574571.926117:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574571.926119:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574571.926124:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574571.926129:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926140:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574571.926144:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926153:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574571.926156:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574571.926162:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574571.926167:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574571.926170:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306176441
00000100:00000010:0.0:1788574571.926176:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574571.926183:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574571.926188:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306176441
00000100:00000010:0.0:1788574571.926192:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d72a0.
00000100:00000001:0.0:1788574571.926195:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574571.926199:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574571.926201:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574571.926204:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574571.926209:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574571/real 1788574571]  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574571.926219:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574571.926221:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574571.926223:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574571.926226:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574571.926228:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574571.926231:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574571.926235:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574571.926245:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926254:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574571.926256:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.926259:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574571.926261:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574571.926274:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926283:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574571.926288:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926297:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574571.926301:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.926303:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574571.926304:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.926307:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574571.926309:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574571.926312:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574571.926350:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574571.926353:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574571.926358:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574571.926368:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393144448:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574571.926374:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.926376:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574571.926380:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393144448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926419:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574571.926422:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574571.926425:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574571.926429:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574571.926432:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574571.926434:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574571.926436:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574571.926439:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788574571.926443:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574571.926445:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574571.926447:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:0.0:1788574571.926450:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574571.926452:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.926454:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.926461:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574571.926463:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574571.926466:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574571.926467:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574571.926469:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574571.926473:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574571.926482:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574571.926484:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574571.926486:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574571.926489:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574571.926492:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144576:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574571.926495:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574571.926498:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574571.926499:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574571.926501:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574571.926505:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574571.926507:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574571.926511:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574571.926514:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574571.926518:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574571.926521:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393144576, portal 10
00000100:00000001:0.0:1788574571.926523:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574571.926525:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:0.0:1788574571.926531:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926538:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574571.926541:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574571.926543:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574571.926546:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393144576, offset 0
00000400:00000200:0.0:1788574571.926550:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574571.926554:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574571.926558:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574571.926562:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574571.926565:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574571.926568:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574571.926570:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574571.926574:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330385:330385:256:4294967295] 192.168.202.110@tcp LPNI seq info [330342:330342:8:4294967295]
00000400:00000200:0.0:1788574571.926579:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574571.926587:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574571.926591:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574571.926594:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574571.926599:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574571.926602:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574571.926606:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574571.926609:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574571.926614:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574571.926616:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574571.926619:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574571.926623:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574571.926626:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574571.926628:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574571.926632:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574571.926636:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926644:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574571.926648:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926655:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574571.926657:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574571.926660:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574571.926663:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574571.926667:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574571.926669:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306175527
00000100:00000001:0.0:1788574571.926672:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306176441
00000100:00000001:0.0:1788574571.926675:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306175527
00000100:00000001:0.0:1788574571.926677:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306176441
00000100:00000001:0.0:1788574571.926680:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574571.926682:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574571.926684:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574571.926688:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574571/real 1788574571]  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574571.926696:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574571.926698:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574571.926700:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574571.926702:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574571.926704:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574571.926707:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574571.926710:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574571.926719:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926732:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574571.926733:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.926735:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574571.926736:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574571.926740:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926748:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574571.926752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926759:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574571.926762:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.926763:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574571.926765:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.926767:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574571.926769:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574571.926772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574571.926775:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574571.926778:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574571.926782:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574571.926790:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144576:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574571.926794:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574571.926797:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926805:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574571.926806:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574571.926809:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574571.926812:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574571.926814:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574571.926817:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574571.926820:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574571.926822:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574571.926824:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574571.926826:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574571.926829:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574571.926831:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574571.926834:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574571.926837:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574571.926841:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574571.926844:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393144704, portal 4
00000100:00000001:0.0:1788574571.926846:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574571.926847:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:0.0:1788574571.926854:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926862:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574571.926865:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574571.926867:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574571.926870:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393144704, offset 0
00000400:00000200:0.0:1788574571.926874:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574571.926878:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574571.926882:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574571.926885:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574571.926887:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574571.926890:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574571.926893:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574571.926896:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330386:330386:256:4294967295] 192.168.202.110@tcp LPNI seq info [330343:330343:8:4294967295]
00000400:00000200:0.0:1788574571.926901:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574571.926908:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574571.926913:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574571.926917:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574571.926921:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574571.926924:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574571.926929:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574571.926932:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574571.926936:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574571.926938:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574571.926942:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574571.926946:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574571.926949:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574571.926951:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574571.926953:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574571.926957:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926966:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574571.926970:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.926978:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574571.926980:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574571.926982:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574571.926986:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574571.926990:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574571.926992:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306175673
00000100:00000001:0.0:1788574571.926995:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306176441
00000100:00000001:0.0:1788574571.926997:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306175673
00000100:00000001:0.0:1788574571.927000:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306176441
00000100:00000001:0.0:1788574571.927002:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574571.927005:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574571.927007:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574571.927011:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574571/real 1788574571]  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574571.927021:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574571.927023:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574571.927025:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574571.927027:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574571.927029:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574571.927031:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574571.927035:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574571.927044:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.927051:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574571.927052:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.927054:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574571.927055:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574571.927059:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.927067:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574571.927071:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.927079:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574571.927082:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.927084:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574571.927085:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.927087:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574571.927089:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574571.927092:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574571.927095:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574571.927098:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574571.927103:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574571.927112:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574571.927116:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.927119:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574571.927122:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393144576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.927130:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574571.927131:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574571.927133:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574571.927136:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574571.927138:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574571.927140:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574571.927142:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574571.927144:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788574571.927147:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574571.927148:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574571.927150:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:0.0:1788574571.927153:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574571.927154:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.927156:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574571.927160:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393144704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574571.927168:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574571.927170:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574571.927171:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574571.927174:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574571.927175:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574571.927176:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574571.927178:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574571.927180:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:0.0:1788574571.927182:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574571.927183:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574571.927185:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:0.0:1788574571.927188:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574571.927189:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.927191:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574571.927194:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574571.927196:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574571.927198:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574571.927200:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574572.885500:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306176441, at 4306176448
00000100:00000040:1.0:1788574572.885500:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306176441, at 4306176448
00000100:00000010:0.0:1788574572.885518:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574572.885519:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d72a0.
00000100:00080000:0.0:1788574577.045510:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574577.045520:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574577.045522:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574577.045525:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574577.045529:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574577.045534:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11508
00000400:00000200:0.0:1788574577.045542:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574577.045547:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574577.045549:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574577.045551:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045554:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574577.045556:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045557:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574577.045560:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574577.045561:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574577.045564:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574577.045566:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574577.045567:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574577.045569:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574577.045571:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045572:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574577.045574:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045576:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574577.045578:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574577.045581:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045583:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574577.045585:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574577.045595:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574577.045599:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574577.045601:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574577.045604:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574577.045613:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788574577.045617:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574577.045622:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574577.045624:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574577.045627:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574577.045630:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574577.045634:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574577.045642:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574577.045647:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574577.045655:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574577.045663:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574577.045668:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574577.045671:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574577.045693:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574577.045695:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574577.045698:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574577.045699:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574577.045700:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574577.045702:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574577.045705:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11508
00000400:00000200:0.0:1788574577.045708:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574577.045710:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574577.045712:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574577.045714:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045715:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574577.045717:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045719:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574577.045721:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574577.045722:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574577.045723:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574577.045725:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574577.045726:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574577.045728:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574577.045730:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045731:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574577.045732:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045734:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574577.045736:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:1.0:1788574577.045736:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1788574577.045741:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045743:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574577.045743:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
02000000:00000001:0.0:1788574577.045747:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1788574577.045748:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:0.0:1788574577.045752:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574577.045755:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574577.045757:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:1.0:1788574577.045760:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1788574577.045761:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574577.045767:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574577.045771:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574577.045776:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788574577.045779:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:1.0:1788574577.045782:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788574577.045783:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574577.045786:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574577.045786:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574577.045791:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574577.045792:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574577.045794:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574577.045794:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:0.0:1788574577.045800:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:1.0:1788574577.045800:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574577.045806:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574577.045806:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574577.045814:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1788574577.045815:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393144832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574577.045826:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1788574577.045827:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574577.045830:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574577.045832:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1788574577.045834:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788574577.045837:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574577.045843:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788574577.045846:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574577.045849:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000010:1.0:1788574577.045851:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759800.
00000100:00080000:0.0:1788574577.045855:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
02000000:00000001:1.0:1788574577.045856:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574577.045859:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574577.045863:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1788574577.045866:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574577.045868:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574577.045873:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11508
00000400:00000010:1.0:1788574577.045873:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d268.
00000400:00000200:0.0:1788574577.045881:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574577.045883:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2aa0.
00000400:00000200:0.0:1788574577.045886:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574577.045890:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000200:1.0:1788574577.045892:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393144832, portal 25
00000100:00000040:0.0:1788574577.045894:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574577.045897:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574577.045899:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1788574577.045901:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523503232 : -130484186048384 : ffff895348711880)
00000100:00000040:0.0:1788574577.045904:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045909:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574577.045914:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:1.0:1788574577.045914:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574602 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788574577.045916:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574577.045919:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574577.045922:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574577.045926:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574577.045931:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:1.0:1788574577.045934:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574577.045937:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045940:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574577.045942:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788574577.045944:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045948:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:1.0:1788574577.045948:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000020:00000040:0.0:1788574577.045954:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000200:1.0:1788574577.045955:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393144832, offset 0
00000100:00080000:0.0:1788574577.045957:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574577.045961:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574577.045962:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574577.045965:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1788574577.045969:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788574577.045970:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574577.045974:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574577.045976:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574577.045977:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574577.045979:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574577.045983:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1788574577.045985:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788574577.045988:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574577.045990:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574577.045993:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574577.045993:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788574577.045997:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574577.045999:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574577.046001:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574577.046005:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574577.046008:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574577.046009:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:0.0:1788574577.046014:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1788574577.046018:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1788574577.046024:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574577.046027:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574577.046027:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1788574577.046032:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574577.046035:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788574577.046044:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574577.046045:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330387:330387:256:4294967295] 192.168.202.110@tcp LPNI seq info [330344:330344:8:4294967295]
00000100:00000040:0.0:1788574577.046048:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574577.046054:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:1.0:1788574577.046058:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788574577.046061:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11518)
00000400:00000200:1.0:1788574577.046072:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574577.046080:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574577.046087:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574577.046094:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574577.046099:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574577.046114:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574577.046119:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574577.046122:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306181561
00000100:00000010:1.0:1788574577.046128:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9960.
00000100:00000001:1.0:1788574577.046135:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574577.046153:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306181561
00000100:00000010:1.0:1788574577.046158:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9e40.
00000100:00000001:1.0:1788574577.046162:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574577.046167:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574577.046171:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574577.046176:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574577.046179:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574577.046186:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574577.046188:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1788574577.046191:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574577.046193:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574577.046195:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574577.046200:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574577.046209:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574577.046210:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574577.046213:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574577.046215:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574577.046217:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574577.046220:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574577.046224:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574577.046226:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574577.046227:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574577.046229:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574577.046233:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574577.046236:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574577.046240:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574577.046243:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5da50.
00000400:00000010:1.0:1788574577.046247:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2d48.
00000100:00000200:1.0:1788574577.046251:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393144960, portal 10
00000100:00000001:1.0:1788574577.046253:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574577.046254:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:1.0:1788574577.046262:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574602 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574577.046270:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574577.046273:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574577.046276:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574577.046278:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393144960, offset 0
00000400:00000200:1.0:1788574577.046311:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574577.046317:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574577.046321:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574577.046324:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574577.046328:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574577.046331:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574577.046334:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574577.046339:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330388:330388:255:4294967295] 192.168.202.110@tcp LPNI seq info [330345:330345:7:4294967295]
00000400:00000200:1.0:1788574577.046344:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574577.046351:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574577.046356:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574577.046369:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cee00.
00000800:00000200:1.0:1788574577.046375:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574577.046379:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574577.046383:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574577.046386:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574577.046414:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574577.046417:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306180647
00000100:00000001:1.0:1788574577.046419:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306181561
00000100:00000001:1.0:1788574577.046421:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306180647
00000100:00000001:1.0:1788574577.046422:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306181561
00000100:00000001:1.0:1788574577.046424:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574577.046432:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574577.046435:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574577.046439:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574577.046453:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574577.046455:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574577.046457:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574577.046459:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574577.046461:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574577.046464:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145088:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574577.046467:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574577.046470:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574577.046471:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574577.046473:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574577.046477:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574577.046479:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574577.046483:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574577.046485:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5dec8.
00000400:00000010:1.0:1788574577.046488:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2e58.
00000100:00000200:1.0:1788574577.046490:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393145088, portal 4
00000100:00000001:1.0:1788574577.046492:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574577.046493:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523511296 : -130484186040320 : ffff895348713800)
00000100:00000040:1.0:1788574577.046498:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574602 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574577.046502:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574577.046504:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574577.046505:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574577.046507:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393145088, offset 0
00000400:00000200:1.0:1788574577.046509:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574577.046512:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574577.046514:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574577.046517:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574577.046518:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574577.046520:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574577.046522:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574577.046524:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330389:330389:254:4294967295] 192.168.202.110@tcp LPNI seq info [330346:330346:6:4294967295]
00000400:00000200:1.0:1788574577.046527:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574577.046531:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574577.046534:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574577.046537:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574577.046539:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574577.046542:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574577.046544:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574577.046629:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574577.046639:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574577.046653:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306180793
00000100:00000001:1.0:1788574577.046656:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306181561
00000100:00000001:1.0:1788574577.046659:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306180793
00000100:00000001:1.0:1788574577.046661:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306181561
00000100:00000001:1.0:1788574577.046663:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574577.046665:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574577.046668:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574577.046670:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574577.046674:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574577.046675:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1788574577.046678:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574577.046679:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574577.046681:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574577.046683:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574578.005491:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306181561, at 4306181568
00000100:00000040:1.0:1788574578.005491:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306181561, at 4306181568
00000100:00000010:0.0:1788574578.005506:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9960.
00000100:00000010:1.0:1788574578.005508:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9e40.
00000400:00000100:1.0:1788574580.117706:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788574580.117752:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788574580.117761:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574580.117765:0:2349:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788574580.117770:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574580.117775:0:2349:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788574580.117780:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574580.117784:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574580.117789:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574580.117794:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574580.117807:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:1.0:1788574580.117809:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:1.0:1788574580.117815:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574580.117823:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574602 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574580.117850:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574580.117856:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574602 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574580.117864:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574580.117866:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574580.117875:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574580.117878:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cee00.
00000400:00000200:1.0:1788574580.117882:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574580.117886:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574580.117889:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574580.117892:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574580.117895:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574580.117899:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574602 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.117903:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574580.117910:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574580.117911:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1788574580.117915:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:1.0:1788574580.117917:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574602 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574580.117925:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574580.117926:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574580.117931:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574580.117933:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574580.117936:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00100000:0.0:1788574580.117941:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574577/real 1788574580]  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574602 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574580.117942:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574580.117948:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574580.117951:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574580.117956:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574580.117964:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574602 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574580.117969:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574580.117974:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:1.0:1788574580.117975:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000400:00000010:0.0:1788574580.117980:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5da50.
00000100:00000040:1.0:1788574580.117981:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574602 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788574580.117987:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2d48
00000400:00000010:0.0:1788574580.117992:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2d48.
00000100:00000001:1.0:1788574580.117993:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574580.117997:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574580.117999:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574580.118007:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574580.118019:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118027:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574580.118030:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118033:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574580.118036:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574580.118041:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118050:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574580.118056:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118072:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574580.118076:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118078:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574580.118079:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118082:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574580.118085:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574580.118089:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574580.118095:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574580.118098:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574580.118102:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574580.118112:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393144960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574580.118118:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574580.118122:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574577/real 1788574580]  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574580.118130:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574580.118131:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574580.118134:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5dec8.
00000400:00000200:0.0:1788574580.118137:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2e58
00000400:00000010:0.0:1788574580.118139:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2e58.
00000100:00000001:0.0:1788574580.118141:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574580.118145:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574580.118154:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118161:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574580.118163:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118165:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574580.118167:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574580.118171:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118178:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574580.118182:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118188:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574580.118191:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118192:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574580.118194:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118196:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574580.118198:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574580.118200:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574580.118205:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574580.118207:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574580.118211:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574580.118220:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145088:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574580.118225:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574580.118229:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574577/real 1788574580]  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574580.118236:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574580.118238:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574580.118240:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d268.
00000400:00000200:0.0:1788574580.118243:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2aa0
00000400:00000010:0.0:1788574580.118245:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2aa0.
00000100:00000001:0.0:1788574580.118249:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574580.118252:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574580.118261:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118268:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574580.118270:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118289:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574580.118291:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574580.118296:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118303:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574580.118308:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118313:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574580.118316:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118318:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574580.118319:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118321:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574580.118323:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574580.118326:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574580.118330:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574580.118333:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574580.118336:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574580.118345:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393144832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574580.118349:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118352:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574580.118356:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393144960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118363:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574580.118367:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574580.118371:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975ac00.
02000000:00000001:0.0:1788574580.118376:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574580.118379:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574580.118380:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574580.118383:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574580.118386:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574580.118389:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574580.118435:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574580.118439:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:0.0:1788574580.118444:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574580.118446:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118448:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574580.118452:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713800 x1875444393145088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118458:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574580.118460:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574580.118461:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:0.0:1788574580.118464:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574580.118466:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574580.118467:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574580.118469:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574580.118472:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574580.118475:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574580.118476:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574580.118477:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713800.
00000100:00000001:0.0:1788574580.118479:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574580.118481:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118482:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574580.118486:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711880 x1875444393144832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574602 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574580.118492:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574580.118494:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574580.118496:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759800.
02000000:00000001:0.0:1788574580.118498:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574580.118500:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574580.118501:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574580.118503:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574580.118504:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574580.118507:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574580.118508:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574580.118509:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711880.
00000100:00000001:0.0:1788574580.118512:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574580.118513:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118516:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574580.118524:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574580.118526:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574580.118528:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574580.118531:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574582.165501:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574582.165510:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574582.165512:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574582.165515:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574582.165519:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574582.165525:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11513
00000400:00000200:0.0:1788574582.165533:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574582.165539:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574582.165541:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574582.165543:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574582.165546:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574582.165548:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574582.165550:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574582.165552:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574582.165554:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574582.165556:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574582.165558:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574582.165560:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574582.165562:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574582.165564:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574582.165566:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574582.165573:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574582.165575:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574582.165577:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574582.165581:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574582.165584:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574582.165587:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574582.165590:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574582.165593:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574582.165596:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574582.165598:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574582.165605:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788574582.165608:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574582.165613:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574582.165614:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574582.165617:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574582.165620:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574582.165624:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574582.165631:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574582.165638:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574582.165648:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574582.165659:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574582.165663:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574582.165669:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574582.165696:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574582.165700:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574582.165704:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574582.165705:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574582.165707:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574582.165710:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574582.165713:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11513
00000400:00000200:0.0:1788574582.165717:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574582.165721:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574582.165724:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574582.165728:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574582.165731:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574582.165734:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574582.165737:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:1.0:1788574582.165739:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788574582.165740:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574582.165744:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:1.0:1788574582.165746:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788574582.165748:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1788574582.165750:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1788574582.165752:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574582.165757:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574582.165761:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574582.165761:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574582.165770:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574582.165776:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574582.165781:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574582.165783:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574582.165786:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:1.0:1788574582.165787:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788574582.165792:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:1.0:1788574582.165795:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1788574582.165800:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1788574582.165801:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574582.165806:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574582.165807:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574582.165811:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1788574582.165816:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393145216:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788574582.165821:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574582.165822:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574582.165826:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574582.165830:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1788574582.165831:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1788574582.165836:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1788574582.165838:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1788574582.165841:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:0.0:1788574582.165844:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574582.165848:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574582.165850:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975a000.
00000100:00000010:0.0:1788574582.165853:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
02000000:00000001:1.0:1788574582.165855:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788574582.165859:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574582.165866:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1788574582.165866:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788574582.165870:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574582.165873:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:1.0:1788574582.165874:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
02000000:00000001:0.0:1788574582.165876:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574582.165881:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1788574582.165885:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
02000000:00000010:0.0:1788574582.165887:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574582.165894:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1788574582.165895:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393145216, portal 25
00000100:00000001:1.0:1788574582.165901:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574582.165905:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523503232 : -130484186048384 : ffff895348711880)
00000100:00100000:0.0:1788574582.165910:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1788574582.165914:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.165931:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574582.165935:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574582.165939:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574582.165942:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393145216, offset 0
00000100:00000001:0.0:1788574582.165950:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1788574582.165951:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788574582.165956:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574582.165965:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788574582.165966:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574582.165976:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788574582.165982:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574582.165987:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1788574582.165988:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574582.165995:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:1.0:1788574582.165995:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1788574582.166000:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1788574582.166003:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:0.0:1788574582.166004:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574582.166011:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:1.0:1788574582.166011:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:0.0:1788574582.166017:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11513
00000400:00000200:1.0:1788574582.166021:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330390:330390:256:4294967295] 192.168.202.110@tcp LPNI seq info [330347:330347:8:4294967295]
00000400:00000200:1.0:1788574582.166029:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574582.166040:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1788574582.166049:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.0:1788574582.166051:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574582.166060:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574582.166068:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:0.0:1788574582.166071:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:1.0:1788574582.166075:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00080000:0.0:1788574582.166076:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574582.166084:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:1.1:1788574582.166090:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574582.166091:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574582.166097:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:1.1:1788574582.166098:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1788574582.166102:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000100:1.0:1788574582.166107:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788574582.166109:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000010:1.0:1788574582.166111:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000020:00000040:0.0:1788574582.166112:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574582.166117:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1788574582.166118:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:0.0:1788574582.166122:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:1.0:1788574582.166124:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1788574582.166128:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:1.0:1788574582.166132:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000020:00000001:0.0:1788574582.166133:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:1.0:1788574582.166136:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000040:0.0:1788574582.166141:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574582.166143:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:0.0:1788574582.166145:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1788574582.166150:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574582.166153:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574582.166159:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574582.166165:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:1.0:1788574582.166167:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1788574582.166171:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:1.0:1788574582.166176:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574607 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574582.166177:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574582.166181:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574582.166186:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574582.166191:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574582.166193:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574582.166195:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574582.166197:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574582.166199:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574582.166203:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788574582.166206:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:1.0:1788574582.166207:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574582.166210:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1788574582.166212:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:0.0:1788574582.166213:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574582.166217:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574582.166219:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574582.166221:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574582.166223:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574582.166227:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000400:00000200:1.0:1788574582.166232:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574582.166233:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574582.166238:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306186681
00000100:00100000:0.0:1788574582.166243:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:1.0:1788574582.166247:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9e40.
00000100:00000001:1.0:1788574582.166258:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574582.166279:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306186681
00000100:00000010:1.0:1788574582.166283:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9780.
00000100:00000001:1.0:1788574582.166286:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574582.166289:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574582.166295:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574582.166297:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574582.166300:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1788574582.166302:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574582.166309:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574582/real 1788574582]  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574582.166311:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574582.166337:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1788574582.166337:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1788574582.166342:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:1.0:1788574582.166342:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574582.166345:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000100:00000040:0.0:1788574582.166352:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:1.0:1788574582.166352:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574582.166355:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574582.166360:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000040:0.0:1788574582.166362:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11523)
00000100:00000200:1.0:1788574582.166366:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574582.166377:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166384:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574582.166386:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.166389:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574582.166434:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574582.166441:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166449:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574582.166454:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166461:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574582.166464:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.166466:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574582.166468:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.166470:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574582.166472:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574582.166476:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574582.166480:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574582.166482:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574582.166487:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574582.166495:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393145216:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574582.166500:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.166502:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574582.166505:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711880 x1875444393145216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166512:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574582.166515:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574582.166518:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975a000.
02000000:00000001:1.0:1788574582.166522:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574582.166524:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574582.166525:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574582.166527:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574582.166530:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:1.0:1788574582.166534:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574582.166536:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574582.166538:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711880.
00000100:00000001:1.0:1788574582.166541:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574582.166542:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.166544:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.166552:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574582.166554:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574582.166556:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574582.166558:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574582.166559:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574582.166563:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166569:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574582.166571:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574582.166573:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574582.166575:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574582.166577:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574582.166580:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145344:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574582.166583:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574582.166586:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574582.166587:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574582.166589:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574582.166593:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975a000.
02000000:00000001:1.0:1788574582.166594:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574582.166599:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574582.166602:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574582.166606:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574582.166610:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393145344, portal 10
00000100:00000001:1.0:1788574582.166612:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574582.166613:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523511296 : -130484186040320 : ffff895348713800)
00000100:00000040:1.0:1788574582.166620:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166634:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574582.166636:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574582.166640:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574582.166642:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393145344, offset 0
00000400:00000200:1.0:1788574582.166647:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574582.166652:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574582.166657:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574582.166660:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574582.166663:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574582.166666:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574582.166669:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574582.166672:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330391:330391:256:4294967295] 192.168.202.110@tcp LPNI seq info [330348:330348:8:4294967295]
00000400:00000200:1.0:1788574582.166676:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574582.166683:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574582.166688:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574582.166692:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574582.166696:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574582.166699:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1788574582.166704:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574582.166707:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1788574582.166710:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574582.166711:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574582.166714:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574582.166716:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574582.166719:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574582.166720:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574582.166722:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574582.166725:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166731:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574582.166733:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574607 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166737:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574582.166738:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574582.166741:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574582.166744:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574582.166747:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574582.166748:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306185767
00000100:00000001:1.0:1788574582.166750:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306186681
00000100:00000001:1.0:1788574582.166752:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306185767
00000100:00000001:1.0:1788574582.166754:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306186681
00000100:00000001:1.0:1788574582.166755:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574582.166757:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574582.166759:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574582.166763:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574582/real 1788574582]  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574582.166771:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574582.166773:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574582.166775:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574582.166778:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574582.166779:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574582.166781:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574582.166785:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574582.166794:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166802:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574582.166803:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.166805:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574582.166807:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574582.166811:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166817:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574582.166820:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166825:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574582.166827:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.166828:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574582.166829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.166831:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574582.166832:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574582.166835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574582.166838:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574582.166840:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574582.166844:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574582.166853:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145344:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574582.166857:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574582.166860:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166866:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574582.166867:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574582.166869:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574582.166871:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574582.166873:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574582.166876:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145472:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574582.166879:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574582.166881:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574582.166883:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574582.166885:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574582.166888:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574582.166890:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574582.166894:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574582.166898:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574582.166901:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574582.166904:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393145472, portal 4
00000100:00000001:1.0:1788574582.166907:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574582.166908:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:1.0:1788574582.166915:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.166922:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574582.166924:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574582.166926:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574582.166929:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393145472, offset 0
00000400:00000200:1.0:1788574582.166933:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574582.166937:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574582.166941:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574582.166944:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574582.166946:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574582.166949:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574582.166951:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574582.166955:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330392:330392:256:4294967295] 192.168.202.110@tcp LPNI seq info [330349:330349:8:4294967295]
00000400:00000200:1.0:1788574582.166958:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574582.166964:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574582.166969:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574582.166972:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574582.166976:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574582.166979:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1788574582.166984:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574582.166986:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1788574582.166990:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574582.166992:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574582.166995:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574582.166998:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574582.167001:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574582.167003:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574582.167006:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574582.167010:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.167017:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574582.167021:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574607 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.167028:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574582.167030:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574582.167033:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574582.167036:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574582.167039:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574582.167041:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306185913
00000100:00000001:1.0:1788574582.167043:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306186681
00000100:00000001:1.0:1788574582.167045:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306185913
00000100:00000001:1.0:1788574582.167047:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306186681
00000100:00000001:1.0:1788574582.167049:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574582.167052:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574582.167053:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574582.167058:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574582/real 1788574582]  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574582.167066:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574582.167068:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574582.167070:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574582.167072:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574582.167074:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574582.167077:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574582.167080:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574582.167088:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.167101:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574582.167103:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.167105:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574582.167106:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574582.167110:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.167118:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574582.167122:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.167130:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574582.167132:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.167133:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574582.167134:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.167136:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574582.167137:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574582.167139:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574582.167141:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574582.167143:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574582.167145:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574582.167151:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145472:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574582.167153:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.167155:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574582.167156:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713800 x1875444393145344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.167160:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574582.167162:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574582.167164:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975a000.
02000000:00000001:1.0:1788574582.167166:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574582.167167:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574582.167168:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574582.167169:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574582.167171:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:1.0:1788574582.167173:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574582.167174:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574582.167175:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713800.
00000100:00000001:1.0:1788574582.167177:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574582.167178:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.167179:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574582.167181:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393145472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574607 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574582.167185:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574582.167186:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574582.167187:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574582.167188:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574582.167189:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574582.167190:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574582.167191:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574582.167192:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:1.0:1788574582.167193:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574582.167193:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574582.167194:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:1.0:1788574582.167195:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574582.167197:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.167198:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574582.167201:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574582.167202:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574582.167204:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574582.167205:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574583.125508:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306186681, at 4306186688
00000100:00000040:0.0:1788574583.125509:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306186681, at 4306186688
00000100:00000010:0.0:1788574583.125523:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9e40.
00000100:00000010:1.0:1788574583.125526:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9780.
00000100:00080000:0.0:1788574587.285504:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574587.285567:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574587.285570:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574587.285575:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574587.285580:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574587.285588:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11518
00000400:00000200:0.0:1788574587.285597:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574587.285602:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574587.285605:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574587.285608:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285610:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574587.285612:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285614:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574587.285617:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574587.285618:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574587.285620:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574587.285623:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574587.285624:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574587.285627:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574587.285629:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285630:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574587.285632:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285633:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574587.285635:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574587.285637:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285639:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574587.285642:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574587.285644:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574587.285646:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574587.285648:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574587.285650:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574587.285658:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574587.285660:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574587.285663:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574587.285664:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574587.285666:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574587.285669:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574587.285672:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574587.285678:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574587.285684:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574587.285691:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574587.285699:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574587.285703:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574587.285707:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574587.285723:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574587.285726:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574587.285728:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574587.285730:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574587.285731:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574587.285733:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574587.285735:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11518
00000400:00000200:0.0:1788574587.285738:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574587.285740:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574587.285742:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574587.285744:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285745:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574587.285747:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285748:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574587.285749:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574587.285750:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574587.285751:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574587.285753:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574587.285754:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574587.285756:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574587.285758:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285760:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574587.285762:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285763:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574587.285765:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574587.285767:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574587.285769:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574587.285770:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574587.285772:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574587.285776:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574587.285777:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574587.285779:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574587.285782:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574587.285784:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574587.285786:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574587.285788:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574587.285792:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:1.0:1788574587.285793:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
02000000:00000001:0.0:1788574587.285794:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574587.285797:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574587.285799:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1788574587.285799:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:0.0:1788574587.285802:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574587.285805:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574587.285809:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788574587.285812:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574587.285819:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1788574587.285821:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574587.285839:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574587.285844:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.285845:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574587.285849:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574587.285853:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788574587.285855:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574587.285860:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574587.285867:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574587.285868:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574587.285871:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574587.285876:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00100000:1.0:1788574587.285877:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393145600:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574587.285878:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574587.285881:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574587.285886:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:1.0:1788574587.285888:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:0.0:1788574587.285891:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11518
00000400:00000200:0.0:1788574587.285895:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788574587.285895:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000400:00000200:0.0:1788574587.285898:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574587.285899:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574587.285902:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285903:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574587.285907:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:1.0:1788574587.285908:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574587.285909:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574587.285913:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574587.285915:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
02000000:00000001:1.0:1788574587.285915:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:0.0:1788574587.285919:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574587.285920:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574587.285922:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
02000000:00000010:1.0:1788574587.285923:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
00000020:00000001:0.0:1788574587.285924:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
02000000:00000001:1.0:1788574587.285930:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574587.285931:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285934:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574587.285937:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285939:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574587.285942:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:1.0:1788574587.285943:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788574587.285947:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574587.285950:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1788574587.285951:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
02000000:00000001:0.0:1788574587.285952:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574587.285955:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574587.285959:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574587.285960:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1788574587.285961:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000008:00100000:0.0:1788574587.285962:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574587.285967:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574587.285969:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1788574587.285972:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393145600, portal 25
00000100:00000010:0.0:1788574587.285973:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788574587.285976:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:1.0:1788574587.285979:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574587.285983:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
02000000:00000001:0.0:1788574587.285989:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574587.285992:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574587.285993:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574587.285994:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574587.285997:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574587.285999:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574587.286005:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574587.286011:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.286012:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:0.0:1788574587.286020:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1788574587.286020:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574587.286026:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000001:0.0:1788574587.286028:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574587.286032:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1788574587.286034:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393145600, offset 0
00000100:00000040:0.0:1788574587.286037:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574587.286044:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574587.286047:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788574587.286048:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574587.286053:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574587.286057:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11528)
00000400:00000200:1.0:1788574587.286061:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574587.286070:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574587.286077:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574587.286081:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574587.286085:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574587.286089:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574587.286093:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330393:330393:256:4294967295] 192.168.202.110@tcp LPNI seq info [330350:330350:8:4294967295]
00000400:00000200:1.0:1788574587.286099:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574587.286108:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574587.286115:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574587.286121:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574587.286128:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574587.286132:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1788574587.286139:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574587.286143:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1788574587.286147:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574587.286150:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574587.286156:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574587.286161:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574587.286167:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574587.286170:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574587.286175:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574587.286179:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286190:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574587.286194:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286201:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.286203:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574587.286209:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574587.286214:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574587.286217:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306191801
00000100:00000010:1.0:1788574587.286223:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9780.
00000100:00000001:1.0:1788574587.286244:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574587.286257:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306191801
00000100:00000010:1.0:1788574587.286261:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9e40.
00000100:00000001:1.0:1788574587.286265:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574587.286269:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574587.286279:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.286283:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574587.286291:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574587/real 1788574587]  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574587.286301:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574587.286303:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574587.286306:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574587.286309:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574587.286311:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574587.286314:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574587.286317:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574587.286326:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286333:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574587.286335:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.286338:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574587.286339:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574587.286344:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286351:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574587.286357:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286364:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574587.286367:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.286369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574587.286371:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.286373:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574587.286375:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574587.286379:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574587.286383:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574587.286386:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574587.286416:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574587.286426:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393145600:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574587.286432:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.286435:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574587.286438:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393145600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286455:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574587.286458:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574587.286461:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574587.286464:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574587.286466:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574587.286468:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574587.286470:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574587.286473:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:1.0:1788574587.286476:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574587.286478:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574587.286480:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:1.0:1788574587.286483:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574587.286485:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.286487:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.286495:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574587.286497:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.286499:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574587.286501:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574587.286503:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574587.286507:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286515:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574587.286516:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574587.286518:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574587.286520:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574587.286523:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574587.286526:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574587.286530:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574587.286532:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574587.286534:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574587.286535:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574587.286539:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574587.286541:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574587.286545:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574587.286548:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574587.286551:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574587.286554:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393145728, portal 10
00000100:00000001:1.0:1788574587.286556:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574587.286558:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000040:1.0:1788574587.286565:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286571:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574587.286574:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574587.286577:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574587.286579:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393145728, offset 0
00000400:00000200:1.0:1788574587.286583:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574587.286588:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574587.286592:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574587.286595:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574587.286598:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574587.286601:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574587.286603:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574587.286607:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330394:330394:256:4294967295] 192.168.202.110@tcp LPNI seq info [330351:330351:8:4294967295]
00000400:00000200:1.0:1788574587.286612:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574587.286619:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574587.286624:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574587.286628:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574587.286632:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574587.286635:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1788574587.286640:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574587.286643:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1788574587.286648:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574587.286650:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574587.286653:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574587.286657:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574587.286660:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574587.286662:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574587.286665:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574587.286669:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286677:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574587.286680:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286687:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.286689:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574587.286692:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574587.286696:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574587.286700:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574587.286703:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306190887
00000100:00000001:1.0:1788574587.286706:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306191801
00000100:00000001:1.0:1788574587.286708:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306190887
00000100:00000001:1.0:1788574587.286710:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306191801
00000100:00000001:1.0:1788574587.286712:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574587.286715:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.286717:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574587.286721:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574587/real 1788574587]  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574587.286730:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574587.286731:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574587.286733:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574587.286736:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574587.286738:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574587.286740:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574587.286743:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574587.286753:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286760:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574587.286761:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.286763:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574587.286765:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574587.286769:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286776:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574587.286780:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286787:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574587.286790:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.286792:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574587.286793:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.286795:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574587.286797:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574587.286800:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574587.286803:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574587.286806:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574587.286811:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574587.286819:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574587.286823:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574587.286827:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286834:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574587.286836:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574587.286838:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574587.286840:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574587.286842:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574587.286846:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574587.286849:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574587.286851:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574587.286853:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574587.286855:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574587.286859:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975a000.
02000000:00000001:1.0:1788574587.286861:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574587.286865:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574587.286868:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574587.286872:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574587.286875:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393145856, portal 4
00000100:00000001:1.0:1788574587.286878:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574587.286879:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000040:1.0:1788574587.286886:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286893:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574587.286895:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574587.286898:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574587.286901:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393145856, offset 0
00000400:00000200:1.0:1788574587.286906:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574587.286910:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574587.286914:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574587.286917:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574587.286920:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574587.286922:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574587.286925:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574587.286928:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330395:330395:256:4294967295] 192.168.202.110@tcp LPNI seq info [330352:330352:8:4294967295]
00000400:00000200:1.0:1788574587.286933:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574587.286940:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574587.286945:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574587.286949:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574587.286954:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574587.286957:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1788574587.286962:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574587.286964:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1788574587.286968:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574587.286970:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574587.286974:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574587.286978:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574587.286982:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574587.286984:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574587.286986:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574587.286990:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.286998:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574587.287002:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.287008:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.287010:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574587.287014:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574587.287017:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574587.287021:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574587.287024:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306191033
00000100:00000001:1.0:1788574587.287026:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306191801
00000100:00000001:1.0:1788574587.287028:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306191033
00000100:00000001:1.0:1788574587.287030:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306191801
00000100:00000001:1.0:1788574587.287033:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574587.287035:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.287037:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574587.287041:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574587/real 1788574587]  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574587.287050:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574587.287052:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574587.287053:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574587.287056:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574587.287058:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574587.287060:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574587.287063:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574587.287073:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.287080:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574587.287081:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.287083:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574587.287085:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574587.287089:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.287096:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574587.287101:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.287108:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574587.287111:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.287113:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574587.287114:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.287116:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574587.287118:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574587.287121:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574587.287125:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574587.287127:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574587.287131:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574587.287140:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393145856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574587.287144:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.287146:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574587.287150:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393145728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.287156:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574587.287158:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574587.287160:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975b400.
02000000:00000001:1.0:1788574587.287163:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574587.287165:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574587.287166:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574587.287168:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574587.287170:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5c00.
02000000:00000001:1.0:1788574587.287173:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574587.287174:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574587.287176:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:1.0:1788574587.287179:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574587.287180:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.287182:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574587.287185:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393145856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574587.287199:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574587.287201:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574587.287202:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975a000.
02000000:00000001:1.0:1788574587.287205:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574587.287207:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574587.287208:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574587.287210:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574587.287212:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:1.0:1788574587.287214:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574587.287215:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574587.287217:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:1.0:1788574587.287219:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574587.287220:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.287223:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574587.287226:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574587.287228:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574587.287247:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574587.287249:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574588.245467:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306191801, at 4306191808
00000100:00000040:1.0:1788574588.245473:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306191801, at 4306191808
00000100:00000010:0.0:1788574588.245480:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9780.
00000100:00000010:1.0:1788574588.245483:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9e40.
00000100:00080000:1.0:1788574592.405531:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574592.405596:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574592.405600:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574592.405604:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574592.405608:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574592.405615:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11523
00000400:00000200:1.0:1788574592.405624:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574592.405631:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574592.405635:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574592.405638:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574592.405642:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574592.405645:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574592.405649:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574592.405653:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574592.405655:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574592.405658:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574592.405661:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574592.405664:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574592.405666:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574592.405670:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574592.405672:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574592.405674:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574592.405676:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574592.405680:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574592.405683:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574592.405687:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574592.405690:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574592.405693:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574592.405696:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574592.405699:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574592.405702:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574592.405712:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788574592.405716:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574592.405721:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574592.405722:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574592.405726:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574592.405728:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574592.405732:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574592.405740:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574592.405747:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574592.405757:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574592.405769:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574592.405774:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574592.405779:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574592.405804:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574592.405807:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574592.405811:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574592.405814:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574592.405816:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574592.405820:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:0.0:1788574592.405827:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574592.405834:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574592.405837:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00080000:1.0:1788574592.405838:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11523
00000400:00000200:1.0:1788574592.405847:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788574592.405848:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574592.405853:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574592.405859:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574592.405867:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574592.405872:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574592.405874:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574592.405876:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574592.405879:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1788574592.405881:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1788574592.405881:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574592.405885:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574592.405889:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:0.0:1788574592.405890:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574592.405892:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574592.405897:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:0.0:1788574592.405898:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574592.405904:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1788574592.405904:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574592.405909:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1788574592.405911:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574592.405915:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574592.405921:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00100000:0.0:1788574592.405922:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393145984:192.168.202.110@tcp:250:kworker.0
00000020:00000040:1.0:1788574592.405926:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788574592.405933:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:1.0:1788574592.405934:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1788574592.405939:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1788574592.405940:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574592.405944:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574592.405945:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574592.405951:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1788574592.405953:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:0.0:1788574592.405958:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:1.0:1788574592.405958:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574592.405965:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574592.405965:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574592.405972:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1788574592.405978:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:1.0:1788574592.405979:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574592.405983:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574592.405986:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000100:00000010:1.0:1788574592.405990:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000400:00000010:0.0:1788574592.405995:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000020:00000040:1.0:1788574592.405995:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574592.406002:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574592.406004:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574592.406006:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393145984, portal 25
00000100:00000001:1.0:1788574592.406007:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574592.406011:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1788574592.406012:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000001:1.0:1788574592.406015:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406016:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345470336 : -130483364081280 : ffff8953796f5180)
02000000:00000010:1.0:1788574592.406024:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000040:0.0:1788574592.406027:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574592.406031:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406041:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:1.0:1788574592.406043:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1788574592.406048:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574592.406053:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574592.406057:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393145984, offset 0
00000100:00000001:1.0:1788574592.406061:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1788574592.406066:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574592.406066:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574592.406075:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788574592.406077:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574592.406087:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:1.0:1788574592.406093:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574592.406094:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1788574592.406099:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788574592.406101:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574592.406106:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:1.0:1788574592.406106:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574592.406112:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1788574592.406113:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:1.0:1788574592.406117:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1788574592.406118:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330396:330396:256:4294967295] 192.168.202.110@tcp LPNI seq info [330353:330353:8:4294967295]
00000400:00000200:0.0:1788574592.406126:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574592.406126:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574592.406134:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11523
00000400:00000200:0.0:1788574592.406142:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574592.406146:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574592.406150:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000400:00000200:1.0:1788574592.406154:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:0.0:1788574592.406159:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574592.406164:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000400:00000200:1.0:1788574592.406164:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574592.406169:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:0.1:1788574592.406174:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788574592.406180:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:0.1:1788574592.406180:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:1.0:1788574592.406187:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000100:0.0:1788574592.406189:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574592.406192:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000100:00000040:1.0:1788574592.406194:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574592.406212:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1788574592.406214:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574592.406219:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1788574592.406220:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:1.0:1788574592.406223:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:0.0:1788574592.406226:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000020:00000001:1.0:1788574592.406227:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:0.0:1788574592.406230:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000020:00000040:1.0:1788574592.406232:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:0.0:1788574592.406235:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000020:00000001:1.0:1788574592.406236:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574592.406242:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:0.0:1788574592.406243:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788574592.406252:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574592.406258:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574592.406262:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574592.406265:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1788574592.406266:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574592.406271:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000040:0.0:1788574592.406274:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788574592.406277:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574592.406283:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574592.406288:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574592.406293:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574592.406294:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574592.406299:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406300:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574592.406305:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574592.406308:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1788574592.406309:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574592.406315:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574592.406317:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574592.406319:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406321:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306196921
00000100:00000010:1.0:1788574592.406326:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788574592.406330:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574592.406332:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574592.406334:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574592.406336:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574592.406337:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574592.406339:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574592.406344:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
02000000:00000010:1.0:1788574592.406344:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574592.406351:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406354:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00100000:1.0:1788574592.406360:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574592.406361:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306196921
00000100:00000010:0.0:1788574592.406368:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7420.
00000100:00000001:0.0:1788574592.406373:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574592.406379:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574592.406380:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574592.406385:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406386:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406410:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574592.406415:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574592/real 1788574592]  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574592.406423:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574592.406425:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574592.406427:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574592.406430:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574592.406432:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000040:1.0:1788574592.406435:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406436:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574592.406442:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574592.406466:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574592.406467:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788574592.406472:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1788574592.406478:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:1.0:1788574592.406480:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:0.0:1788574592.406481:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406485:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574592.406487:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574592.406490:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11533)
00000100:00100000:0.0:1788574592.406491:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406499:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574592.406503:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406507:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574592.406510:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406511:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574592.406512:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406513:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574592.406515:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574592.406517:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574592.406520:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574592.406521:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574592.406524:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574592.406529:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393145984:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574592.406532:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406534:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574592.406536:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5180 x1875444393145984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406540:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574592.406542:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574592.406544:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574592.406546:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574592.406547:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574592.406548:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574592.406550:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574592.406552:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:0.0:1788574592.406555:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574592.406557:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574592.406558:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5180.
00000100:00000001:0.0:1788574592.406560:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574592.406561:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406563:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406569:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574592.406570:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406571:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574592.406573:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574592.406574:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574592.406576:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406581:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574592.406582:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574592.406584:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574592.406585:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574592.406586:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574592.406588:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146112:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574592.406590:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574592.406592:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574592.406593:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574592.406594:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574592.406596:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574592.406598:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574592.406600:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574592.406602:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574592.406604:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574592.406606:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393146112, portal 10
00000100:00000001:0.0:1788574592.406607:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574592.406608:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345476608 : -130483364075008 : ffff8953796f6a00)
00000100:00000040:0.0:1788574592.406613:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406617:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574592.406619:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574592.406621:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574592.406622:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393146112, offset 0
00000400:00000200:0.0:1788574592.406625:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574592.406628:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574592.406631:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574592.406633:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574592.406635:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574592.406637:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574592.406639:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574592.406641:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330397:330397:256:4294967295] 192.168.202.110@tcp LPNI seq info [330354:330354:8:4294967295]
00000400:00000200:0.0:1788574592.406644:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574592.406649:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574592.406652:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574592.406654:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574592.406657:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574592.406659:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574592.406662:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574592.406663:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574592.406666:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574592.406667:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574592.406669:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574592.406672:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574592.406674:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574592.406675:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574592.406677:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574592.406679:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406684:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574592.406686:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406691:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406692:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574592.406694:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574592.406697:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574592.406699:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574592.406701:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306196007
00000100:00000001:0.0:1788574592.406702:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306196921
00000100:00000001:0.0:1788574592.406704:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306196007
00000100:00000001:0.0:1788574592.406706:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306196921
00000100:00000001:0.0:1788574592.406707:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574592.406709:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406710:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574592.406712:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574592/real 1788574592]  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574592.406717:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574592.406718:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574592.406719:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574592.406720:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574592.406721:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574592.406722:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574592.406724:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574592.406729:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406734:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574592.406735:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406736:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574592.406737:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574592.406739:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406743:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574592.406745:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406750:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574592.406752:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406753:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574592.406754:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406755:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574592.406756:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574592.406758:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574592.406760:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574592.406761:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574592.406763:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574592.406768:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146112:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574592.406770:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574592.406772:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406776:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574592.406777:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574592.406778:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574592.406780:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574592.406781:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574592.406782:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146240:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574592.406784:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574592.406785:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574592.406787:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574592.406788:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574592.406790:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574592.406791:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574592.406793:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574592.406795:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574592.406797:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574592.406799:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393146240, portal 4
00000100:00000001:0.0:1788574592.406801:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574592.406801:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345468544 : -130483364083072 : ffff8953796f4a80)
00000100:00000040:0.0:1788574592.406805:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406810:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574592.406811:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574592.406812:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574592.406814:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393146240, offset 0
00000400:00000200:0.0:1788574592.406817:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574592.406819:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574592.406821:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574592.406823:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574592.406824:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574592.406826:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574592.406828:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574592.406830:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330398:330398:256:4294967295] 192.168.202.110@tcp LPNI seq info [330355:330355:8:4294967295]
00000400:00000200:0.0:1788574592.406833:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574592.406837:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574592.406839:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574592.406841:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574592.406843:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574592.406845:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574592.406848:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574592.406849:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574592.406852:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574592.406853:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574592.406855:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574592.406857:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574592.406859:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574592.406860:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574592.406861:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574592.406864:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406869:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574592.406871:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406875:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406876:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574592.406878:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574592.406880:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574592.406882:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574592.406883:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306196153
00000100:00000001:0.0:1788574592.406885:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306196921
00000100:00000001:0.0:1788574592.406886:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306196153
00000100:00000001:0.0:1788574592.406887:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306196921
00000100:00000001:0.0:1788574592.406889:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574592.406890:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.406891:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574592.406893:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574592/real 1788574592]  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574592.406898:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574592.406899:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574592.406900:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574592.406901:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574592.406902:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574592.406903:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574592.406905:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574592.406910:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406918:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574592.406919:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406921:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574592.406922:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574592.406924:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406928:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574592.406930:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406934:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574592.406935:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406936:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574592.406937:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406938:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574592.406939:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574592.406941:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574592.406943:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574592.406944:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574592.406946:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574592.406952:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146240:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574592.406954:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406955:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574592.406957:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6a00 x1875444393146112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.406963:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574592.406965:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574592.406967:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574592.406969:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574592.406971:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574592.406973:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574592.406975:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574592.406976:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:0.0:1788574592.406979:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574592.406980:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574592.406982:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6a00.
00000100:00000001:0.0:1788574592.406984:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574592.406986:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.406988:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574592.406992:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4a80 x1875444393146240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574592.407000:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574592.407001:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574592.407003:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574592.407005:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574592.407006:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574592.407008:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574592.407010:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574592.407011:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:0.0:1788574592.407014:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574592.407015:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574592.407016:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4a80.
00000100:00000001:0.0:1788574592.407018:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574592.407019:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.407020:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574592.407023:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574592.407024:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574592.407025:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574592.407026:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574593.365477:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306196921, at 4306196928
00000100:00000040:0.0:1788574593.365479:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306196921, at 4306196928
00000100:00000010:1.0:1788574593.365491:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7420.
00000100:00000010:0.0:1788574593.365493:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574597.525762:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574597.525769:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574597.525774:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574597.525783:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.525800:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574597.525803:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574597.525807:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574597.525810:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574597.525815:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574597.525822:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393146368:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574597.525830:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574597.525835:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574597.525838:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574597.525843:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574597.525848:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574597.525853:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574597.525863:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574597.525869:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574597.525876:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574597.525883:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393146368, portal 25
00000100:00000001:0.0:1788574597.525888:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574597.525892:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:0.0:1788574597.525901:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574622 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.525912:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574597.525917:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574597.525920:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574597.525926:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393146368, offset 0
00000400:00000200:0.0:1788574597.525933:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574597.525943:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574597.525951:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574597.525958:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574597.525965:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574597.525971:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574597.525977:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574597.525982:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330399:330399:256:4294967295] 192.168.202.110@tcp LPNI seq info [330356:330356:8:4294967295]
00000400:00000200:0.0:1788574597.525991:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574597.526003:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574597.526016:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574597.526022:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574597.526033:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574597.526039:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574597.526047:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574597.526053:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574597.526064:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574597.526069:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574597.526077:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574597.526082:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574597.526089:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574597.526093:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574597.526100:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574597.526104:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574622 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526126:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574597.526132:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574622 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526145:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574597.526148:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574597.526153:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574597.526159:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574597.526163:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306202041
00000100:00000010:0.0:1788574597.526180:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574597.526186:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574597.526195:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306202041
00000100:00000010:0.0:1788574597.526200:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d74e0.
00000100:00000001:0.0:1788574597.526206:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574597.526211:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574597.526216:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574597.526218:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574597.526223:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574597/real 1788574597]  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574597.526234:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574597.526237:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574597.526240:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574597.526243:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574597.526246:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574597.526250:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574597.526253:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574597.526263:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526273:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574597.526275:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526277:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574597.526280:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574597.526287:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526298:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574597.526303:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526312:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574597.526317:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526320:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574597.526322:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526326:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574597.526329:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574597.526334:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574597.526340:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574597.526342:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574597.526348:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574597.526361:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393146368:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574597.526370:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526373:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574597.526377:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393146368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526385:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574597.526389:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574597.526429:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574597.526432:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574597.526434:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574597.526436:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574597.526437:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574597.526439:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:0.0:1788574597.526442:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574597.526444:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574597.526445:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:0.0:1788574597.526448:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574597.526449:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526450:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526456:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574597.526457:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574597.526459:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574597.526460:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574597.526462:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574597.526465:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526470:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574597.526471:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574597.526473:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574597.526475:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574597.526476:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574597.526478:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146496:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574597.526480:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574597.526482:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574597.526483:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574597.526484:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574597.526487:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574597.526488:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574597.526491:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574597.526493:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574597.526496:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574597.526498:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393146496, portal 10
00000100:00000001:0.0:1788574597.526500:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574597.526501:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:0.0:1788574597.526506:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574622 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526510:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574597.526512:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574597.526514:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574597.526515:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393146496, offset 0
00000400:00000200:0.0:1788574597.526518:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574597.526520:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574597.526523:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574597.526525:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574597.526527:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574597.526528:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574597.526530:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574597.526532:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330400:330400:256:4294967295] 192.168.202.110@tcp LPNI seq info [330357:330357:8:4294967295]
00000400:00000200:0.0:1788574597.526535:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574597.526540:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574597.526543:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574597.526546:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574597.526548:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574597.526550:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574597.526554:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574597.526555:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574597.526558:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574597.526559:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574597.526561:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574597.526564:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574597.526566:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574597.526567:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574597.526569:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574597.526571:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574622 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526576:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574597.526578:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574622 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526582:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574597.526584:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574597.526586:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574597.526588:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574597.526591:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574597.526592:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306201127
00000100:00000001:0.0:1788574597.526594:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306202041
00000100:00000001:0.0:1788574597.526596:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306201127
00000100:00000001:0.0:1788574597.526597:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306202041
00000100:00000001:0.0:1788574597.526599:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574597.526600:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574597.526601:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574597.526604:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574597/real 1788574597]  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574597.526610:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574597.526611:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574597.526612:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574597.526613:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574597.526614:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574597.526615:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574597.526617:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574597.526622:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526627:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574597.526628:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526629:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574597.526630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574597.526632:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526636:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574597.526639:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526643:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574597.526644:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526645:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574597.526646:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526647:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574597.526649:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574597.526650:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574597.526652:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574597.526654:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574597.526656:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574597.526662:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146496:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574597.526665:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574597.526667:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526670:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574597.526671:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574597.526673:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574597.526674:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574597.526675:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574597.526677:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574597.526679:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574597.526680:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574597.526680:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574597.526682:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574597.526684:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574597.526685:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574597.526687:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574597.526689:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574597.526691:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574597.526693:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393146624, portal 4
00000100:00000001:0.0:1788574597.526694:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574597.526695:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:0.0:1788574597.526699:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574622 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526704:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574597.526705:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574597.526707:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574597.526708:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393146624, offset 0
00000400:00000200:0.0:1788574597.526711:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574597.526713:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574597.526715:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574597.526717:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574597.526718:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574597.526720:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574597.526722:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574597.526724:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330401:330401:256:4294967295] 192.168.202.110@tcp LPNI seq info [330358:330358:8:4294967295]
00000400:00000200:0.0:1788574597.526726:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574597.526730:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574597.526733:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574597.526735:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574597.526737:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574597.526739:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574597.526741:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574597.526743:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574597.526745:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574597.526747:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574597.526749:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574597.526751:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574597.526753:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574597.526754:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574597.526755:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574597.526757:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574622 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526762:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574597.526764:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574622 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526767:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574597.526768:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574597.526770:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574597.526772:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574597.526774:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574597.526776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306201273
00000100:00000001:0.0:1788574597.526777:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306202041
00000100:00000001:0.0:1788574597.526779:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306201273
00000100:00000001:0.0:1788574597.526780:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306202041
00000100:00000001:0.0:1788574597.526782:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574597.526783:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574597.526784:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574597.526786:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574597/real 1788574597]  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574597.526790:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574597.526791:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574597.526792:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574597.526793:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574597.526794:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574597.526796:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574597.526798:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574597.526803:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526807:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574597.526808:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526809:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574597.526810:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574597.526812:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526816:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574597.526818:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526822:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574597.526824:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526825:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574597.526826:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526827:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574597.526828:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574597.526830:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574597.526832:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574597.526833:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574597.526836:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574597.526840:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574597.526842:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526843:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574597.526845:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393146496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526849:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574597.526850:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574597.526851:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574597.526853:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574597.526853:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574597.526854:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574597.526855:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574597.526856:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:0.0:1788574597.526858:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574597.526859:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574597.526860:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:0.0:1788574597.526862:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574597.526863:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526864:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574597.526866:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393146624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574622 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574597.526874:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574597.526875:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574597.526876:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574597.526877:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574597.526878:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574597.526878:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574597.526880:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574597.526880:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:0.0:1788574597.526882:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574597.526882:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574597.526883:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788574597.526885:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574597.526885:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526887:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574597.526889:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574597.526889:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574597.526891:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574597.526892:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574598.485496:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306202041, at 4306202048
00000100:00000040:1.0:1788574598.485496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306202041, at 4306202048
00000100:00000010:1.0:1788574598.485514:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d74e0.
00000100:00000010:0.0:1788574598.485519:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000800:00000200:0.0:1788574598.549680:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788574598.549685:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574598.549696:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788574598.549699:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574598.549701:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.645818:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574602.645827:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574602.645832:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574602.645852:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.645883:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574602.645898:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574602.645909:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574602.645917:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574602.645928:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574602.645937:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393146752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574602.645950:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574602.645960:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574602.645971:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574602.645988:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574602.646000:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574602.646005:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574602.646019:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574602.646027:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574602.646043:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574602.646055:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393146752, portal 25
00000100:00000001:0.0:1788574602.646061:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574602.646062:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:0.0:1788574602.646073:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574627 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646115:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574602.646123:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574602.646127:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574602.646163:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393146752, offset 0
00000400:00000200:0.0:1788574602.646180:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574602.646195:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574602.646207:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574602.646220:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574602.646228:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574602.646235:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574602.646244:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574602.646253:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330402:330402:256:4294967295] 192.168.202.110@tcp LPNI seq info [330359:330359:8:4294967295]
00000400:00000200:0.0:1788574602.646272:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574602.646296:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574602.646311:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574602.646323:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574602.646333:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574602.646339:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788574602.646352:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574602.646359:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1788574602.646373:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574602.646380:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574602.646436:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574602.646441:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574602.646446:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574602.646448:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574602.646452:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574602.646456:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574627 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646464:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574602.646467:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574627 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646473:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574602.646475:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574602.646479:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574602.646482:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574602.646484:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306207162
00000100:00000010:0.0:1788574602.646489:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574602.646494:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574602.646501:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306207162
00000100:00000010:0.0:1788574602.646503:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7000.
00000100:00000001:0.0:1788574602.646505:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574602.646507:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574602.646510:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574602.646512:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574602.646515:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574602/real 1788574602]  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574602.646522:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574602.646524:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574602.646526:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574602.646528:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574602.646529:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574602.646531:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574602.646533:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574602.646539:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646544:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574602.646545:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.646547:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574602.646548:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574602.646551:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646555:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574602.646559:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646563:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574602.646565:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.646566:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574602.646567:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.646569:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574602.646570:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574602.646573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574602.646576:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574602.646578:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574602.646580:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574602.646586:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393146752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574602.646591:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.646592:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574602.646594:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393146752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646599:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574602.646600:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574602.646602:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574602.646604:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574602.646606:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574602.646608:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574602.646609:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574602.646611:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:0.0:1788574602.646614:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574602.646615:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574602.646617:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:0.0:1788574602.646619:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574602.646620:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.646622:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.646628:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574602.646629:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574602.646631:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574602.646632:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574602.646633:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574602.646636:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646641:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574602.646642:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574602.646643:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574602.646645:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574602.646646:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574602.646648:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574602.646650:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574602.646652:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574602.646653:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574602.646655:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574602.646656:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574602.646658:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574602.646660:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574602.646662:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574602.646665:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574602.646667:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393146880, portal 10
00000100:00000001:0.0:1788574602.646668:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574602.646669:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000100:00000040:0.0:1788574602.646674:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574627 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646678:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574602.646680:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574602.646681:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574602.646683:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393146880, offset 0
00000400:00000200:0.0:1788574602.646685:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574602.646688:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574602.646690:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574602.646693:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574602.646694:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574602.646696:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574602.646698:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574602.646700:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330403:330403:256:4294967295] 192.168.202.110@tcp LPNI seq info [330360:330360:8:4294967295]
00000400:00000200:0.0:1788574602.646703:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574602.646707:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574602.646710:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574602.646712:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574602.646715:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574602.646717:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788574602.646720:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574602.646721:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1788574602.646724:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574602.646725:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574602.646727:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574602.646729:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574602.646732:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574602.646733:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574602.646734:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574602.646736:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574627 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646742:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574602.646744:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574627 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646748:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574602.646749:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574602.646751:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574602.646754:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574602.646756:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574602.646757:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306206247
00000100:00000001:0.0:1788574602.646759:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306207162
00000100:00000001:0.0:1788574602.646761:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306206247
00000100:00000001:0.0:1788574602.646762:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306207162
00000100:00000001:0.0:1788574602.646763:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574602.646765:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574602.646766:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574602.646768:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574602/real 1788574602]  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574602.646773:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574602.646774:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574602.646775:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574602.646777:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574602.646778:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574602.646779:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574602.646782:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574602.646787:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646791:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574602.646792:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.646793:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574602.646794:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574602.646796:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646800:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574602.646802:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646806:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574602.646808:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.646809:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574602.646810:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.646811:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574602.646812:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574602.646814:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574602.646816:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574602.646818:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574602.646821:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574602.646825:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393146880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574602.646828:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574602.646830:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646839:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574602.646840:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574602.646841:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574602.646843:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574602.646844:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574602.646846:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147008:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574602.646848:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574602.646849:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574602.646850:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574602.646851:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574602.646853:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574602.646854:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574602.646856:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574602.646857:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574602.646859:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574602.646861:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393147008, portal 4
00000100:00000001:0.0:1788574602.646863:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574602.646864:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:0.0:1788574602.646870:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574627 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646877:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574602.646880:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574602.646882:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574602.646884:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393147008, offset 0
00000400:00000200:0.0:1788574602.646888:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574602.646892:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574602.646896:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574602.646899:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574602.646902:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574602.646905:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574602.646907:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574602.646911:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330404:330404:256:4294967295] 192.168.202.110@tcp LPNI seq info [330361:330361:8:4294967295]
00000400:00000200:0.0:1788574602.646916:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574602.646922:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574602.646927:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574602.646930:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574602.646932:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574602.646934:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1788574602.646937:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574602.646939:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1788574602.646941:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574602.646943:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574602.646945:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574602.646947:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574602.646950:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574602.646951:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574602.646953:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574602.646955:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574627 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646961:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574602.646963:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574627 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.646967:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574602.646968:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574602.646971:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574602.646973:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574602.646975:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574602.646977:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306206393
00000100:00000001:0.0:1788574602.646978:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306207162
00000100:00000001:0.0:1788574602.646980:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306206393
00000100:00000001:0.0:1788574602.646981:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306207162
00000100:00000001:0.0:1788574602.646982:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574602.646984:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574602.646986:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574602.646988:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574602/real 1788574602]  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574602.646993:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574602.646994:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574602.646995:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574602.646996:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574602.646998:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574602.646999:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574602.647001:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574602.647006:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.647010:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574602.647011:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.647012:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574602.647013:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574602.647015:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.647019:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574602.647021:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.647025:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574602.647027:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.647028:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574602.647029:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.647030:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574602.647031:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574602.647033:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574602.647036:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574602.647037:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574602.647040:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574602.647045:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147008:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574602.647047:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.647048:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574602.647050:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393146880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.647054:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574602.647055:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574602.647056:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574602.647057:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574602.647058:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574602.647059:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574602.647061:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574602.647062:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:0.0:1788574602.647064:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574602.647064:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574602.647065:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:0.0:1788574602.647067:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574602.647068:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.647069:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574602.647071:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393147008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574627 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574602.647075:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574602.647076:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574602.647077:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574602.647078:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574602.647079:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574602.647080:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574602.647081:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574602.647082:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:0.0:1788574602.647083:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574602.647084:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574602.647085:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788574602.647087:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574602.647088:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.647090:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574602.647092:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574602.647093:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574602.647094:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574602.647095:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574603.605487:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306207162, at 4306207168
00000100:00000040:1.0:1788574603.605487:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306207162, at 4306207168
00000100:00000010:1.0:1788574603.605507:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7000.
00000100:00000010:0.0:1788574603.605509:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574607.765721:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574607.765731:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574607.765735:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574607.765752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.765789:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574607.765794:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574607.765809:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574607.765819:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574607.765829:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574607.765835:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393147136:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574607.765844:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574607.765850:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574607.765856:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574607.765861:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574607.765874:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574607.765878:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574607.765895:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574607.765907:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574607.765924:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574607.765941:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393147136, portal 25
00000100:00000001:0.0:1788574607.765955:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574607.765962:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:0.0:1788574607.765976:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574632 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.765997:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574607.766014:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574607.766024:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574607.766037:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393147136, offset 0
00000400:00000200:0.0:1788574607.766054:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574607.766068:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574607.766085:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574607.766102:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574607.766125:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574607.766133:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574607.766142:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574607.766147:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330405:330405:256:4294967295] 192.168.202.110@tcp LPNI seq info [330362:330362:8:4294967295]
00000400:00000200:0.0:1788574607.766152:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574607.766163:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574607.766168:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574607.766174:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574607.766180:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574607.766184:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788574607.766190:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574607.766193:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1788574607.766196:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574607.766199:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574607.766206:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574607.766210:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574607.766216:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574607.766218:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574607.766222:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574607.766226:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574632 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766235:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574607.766238:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574632 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766244:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574607.766245:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574607.766250:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574607.766253:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574607.766256:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306212281
00000100:00000010:0.0:1788574607.766261:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574607.766268:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574607.766275:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306212281
00000100:00000010:0.0:1788574607.766279:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7000.
00000100:00000001:0.0:1788574607.766282:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574607.766285:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574607.766286:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574607.766289:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574607.766292:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574607/real 1788574607]  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574607.766304:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574607.766306:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574607.766309:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574607.766312:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574607.766314:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574607.766317:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574607.766321:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574607.766331:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766339:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574607.766341:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766344:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574607.766346:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574607.766351:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766357:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574607.766361:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766366:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574607.766368:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574607.766370:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766372:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574607.766374:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574607.766377:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574607.766380:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574607.766382:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574607.766386:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574607.766439:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393147136:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574607.766444:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766446:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574607.766448:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393147136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766453:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574607.766456:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574607.766459:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574607.766462:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574607.766464:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574607.766465:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574607.766467:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574607.766470:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:0.0:1788574607.766474:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574607.766476:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574607.766478:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:0.0:1788574607.766481:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574607.766482:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766484:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766491:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574607.766493:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574607.766495:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574607.766496:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574607.766497:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574607.766500:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766506:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574607.766507:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574607.766509:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574607.766510:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574607.766512:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574607.766514:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147264:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574607.766516:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574607.766518:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574607.766519:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574607.766520:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574607.766523:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574607.766524:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574607.766527:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574607.766529:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574607.766532:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574607.766535:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393147264, portal 10
00000100:00000001:0.0:1788574607.766536:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574607.766537:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:0.0:1788574607.766543:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574632 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766548:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574607.766550:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574607.766552:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574607.766553:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393147264, offset 0
00000400:00000200:0.0:1788574607.766556:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574607.766559:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574607.766562:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574607.766565:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574607.766567:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574607.766568:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574607.766570:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574607.766572:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330406:330406:256:4294967295] 192.168.202.110@tcp LPNI seq info [330363:330363:8:4294967295]
00000400:00000200:0.0:1788574607.766575:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574607.766580:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574607.766583:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574607.766585:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574607.766588:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574607.766590:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788574607.766593:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574607.766595:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1788574607.766599:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574607.766600:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574607.766602:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574607.766604:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574607.766607:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574607.766608:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574607.766610:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574607.766612:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574632 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766618:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574607.766620:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574632 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766624:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574607.766625:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574607.766627:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574607.766631:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574607.766633:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574607.766635:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306211367
00000100:00000001:0.0:1788574607.766637:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306212281
00000100:00000001:0.0:1788574607.766639:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306211367
00000100:00000001:0.0:1788574607.766641:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306212281
00000100:00000001:0.0:1788574607.766642:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574607.766644:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574607.766645:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574607.766647:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574607/real 1788574607]  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574607.766652:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574607.766653:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574607.766654:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574607.766655:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574607.766656:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574607.766657:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574607.766659:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574607.766664:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766669:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574607.766670:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766671:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574607.766672:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574607.766674:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766678:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574607.766680:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766686:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574607.766688:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766689:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574607.766690:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766691:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574607.766693:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574607.766695:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574607.766697:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574607.766698:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574607.766700:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574607.766705:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147264:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574607.766708:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574607.766710:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766715:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574607.766716:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574607.766718:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574607.766719:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574607.766720:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574607.766722:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147392:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574607.766723:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574607.766725:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574607.766726:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574607.766727:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574607.766729:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574607.766730:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574607.766733:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574607.766735:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574607.766737:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574607.766739:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393147392, portal 4
00000100:00000001:0.0:1788574607.766741:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574607.766742:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:0.0:1788574607.766746:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574632 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766749:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574607.766751:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574607.766753:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574607.766754:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393147392, offset 0
00000400:00000200:0.0:1788574607.766757:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574607.766759:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574607.766761:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574607.766763:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574607.766765:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574607.766767:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574607.766769:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574607.766771:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330407:330407:256:4294967295] 192.168.202.110@tcp LPNI seq info [330364:330364:8:4294967295]
00000400:00000200:0.0:1788574607.766774:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574607.766778:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574607.766781:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574607.766782:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574607.766784:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574607.766786:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788574607.766789:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574607.766791:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1788574607.766794:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574607.766795:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574607.766797:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574607.766799:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574607.766801:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574607.766803:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574607.766805:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574607.766807:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574632 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766813:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574607.766815:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574632 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766818:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574607.766819:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574607.766821:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574607.766823:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574607.766825:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574607.766826:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306211513
00000100:00000001:0.0:1788574607.766828:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306212281
00000100:00000001:0.0:1788574607.766830:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306211513
00000100:00000001:0.0:1788574607.766832:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306212281
00000100:00000001:0.0:1788574607.766833:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574607.766834:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574607.766835:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574607.766837:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574607/real 1788574607]  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574607.766842:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574607.766843:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574607.766843:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574607.766845:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574607.766846:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574607.766847:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574607.766849:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574607.766854:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766862:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574607.766864:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766865:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574607.766866:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574607.766868:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766872:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574607.766874:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766878:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574607.766880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766881:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574607.766882:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766883:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574607.766885:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574607.766887:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574607.766890:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574607.766892:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574607.766896:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574607.766904:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147392:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574607.766907:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766909:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574607.766912:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393147264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766920:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574607.766922:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574607.766925:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574607.766927:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574607.766929:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574607.766931:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574607.766933:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574607.766935:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:0.0:1788574607.766938:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574607.766939:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574607.766941:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:0.0:1788574607.766943:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574607.766944:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766945:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574607.766948:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393147392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574632 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574607.766953:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574607.766954:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574607.766955:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574607.766957:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574607.766958:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574607.766959:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574607.766960:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574607.766961:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:0.0:1788574607.766962:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574607.766963:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574607.766964:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:0.0:1788574607.766966:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574607.766966:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766968:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574607.766970:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574607.766971:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574607.766972:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574607.766973:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574608.725471:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306212281, at 4306212288
00000100:00000040:0.0:1788574608.725477:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306212281, at 4306212288
00000100:00000010:1.0:1788574608.725492:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7000.
00000100:00000010:0.0:1788574608.725497:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00080000:1.0:1788574612.885504:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574612.885515:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574612.885517:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574612.885522:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574612.885526:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574612.885533:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11544
00000400:00000200:1.0:1788574612.885541:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574612.885545:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574612.885548:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574612.885551:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885554:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574612.885556:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885557:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574612.885560:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574612.885562:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574612.885564:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574612.885566:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574612.885568:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574612.885569:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574612.885571:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885580:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574612.885581:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885583:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574612.885585:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574612.885588:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885590:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574612.885592:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574612.885594:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574612.885596:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574612.885598:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574612.885600:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574612.885608:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788574612.885610:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574612.885613:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574612.885615:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574612.885617:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574612.885619:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574612.885623:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574612.885629:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574612.885634:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574612.885641:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574612.885649:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574612.885653:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574612.885656:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574612.885672:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574612.885674:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574612.885677:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574612.885678:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574612.885679:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574612.885681:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574612.885683:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11544
00000400:00000200:1.0:1788574612.885686:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574612.885688:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574612.885689:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574612.885692:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885693:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574612.885695:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885697:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574612.885699:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574612.885700:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574612.885701:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574612.885703:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574612.885704:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574612.885705:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574612.885707:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885709:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574612.885710:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885711:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574612.885714:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574612.885716:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574612.885717:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574612.885718:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574612.885722:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788574612.885723:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574612.885726:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1788574612.885727:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574612.885730:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574612.885732:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:0.0:1788574612.885736:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:1.0:1788574612.885737:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574612.885742:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574612.885746:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574612.885750:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000100:00000001:0.0:1788574612.885755:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1788574612.885755:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:0.0:1788574612.885759:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574612.885760:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574612.885763:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574612.885766:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788574612.885766:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574612.885770:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574612.885770:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574612.885774:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574612.885775:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574612.885778:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00100000:0.0:1788574612.885779:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393147520:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574612.885782:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.885785:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574612.885788:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:1.0:1788574612.885789:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1788574612.885791:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574612.885796:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574612.885800:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574612.885803:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574612.885810:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1788574612.885813:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574612.885814:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574612.885818:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000100:00000040:1.0:1788574612.885820:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788574612.885825:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000001:1.0:1788574612.885831:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574612.885834:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393147520, portal 25
00000100:00080000:1.0:1788574612.885835:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1788574612.885839:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:1.0:1788574612.885840:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574612.885843:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345466752 : -130483364084864 : ffff8953796f4380)
00000100:00000001:1.0:1788574612.885844:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574612.885847:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574612.885850:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:0.0:1788574612.885853:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788574612.885855:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11544
00000400:00000200:1.0:1788574612.885863:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574612.885868:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574612.885871:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1788574612.885876:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574612.885876:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885881:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574612.885885:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:0.0:1788574612.885886:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885887:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574612.885891:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574612.885892:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000010:0.0:1788574612.885893:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000020:00000001:1.0:1788574612.885894:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574612.885896:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574612.885900:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:0.0:1788574612.885901:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393147520, offset 0
00000020:00000001:1.0:1788574612.885902:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574612.885908:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885910:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574612.885913:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574612.885916:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574612.885916:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574612.885921:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574612.885924:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1788574612.885927:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574612.885931:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574612.885936:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:1.0:1788574612.885937:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574612.885943:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1788574612.885944:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:1.0:1788574612.885949:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574612.885950:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574612.885955:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1788574612.885955:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1788574612.885960:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000008:00100000:1.0:1788574612.885960:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574612.885967:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574612.885968:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330408:330408:256:4294967295] 192.168.202.110@tcp LPNI seq info [330365:330365:8:4294967295]
00000100:00000001:1.0:1788574612.885972:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574612.885976:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:1.0:1788574612.885980:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000400:00000200:0.0:1788574612.885987:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:1.0:1788574612.885987:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574612.885995:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:0.0:1788574612.885997:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788574612.885999:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788574612.886003:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000100:00000001:1.0:1788574612.886003:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574612.886007:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574612.886011:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788574612.886012:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574612.886018:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
02000000:00000010:1.0:1788574612.886019:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574612.886025:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788574612.886026:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574612.886030:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00100000:1.0:1788574612.886034:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:0.0:1788574612.886037:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574612.886041:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574612.886049:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1788574612.886051:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1788574612.886056:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1788574612.886057:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574612.886062:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574612.886066:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000040:1.0:1788574612.886067:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886070:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574612.886089:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574612.886106:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.886110:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574612.886110:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574612.886116:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574637 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788574612.886118:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574612.886126:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11554)
00000100:00000001:0.0:1788574612.886127:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.886132:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574612.886138:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574612.886141:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574612.886144:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306217401
00000100:00000010:0.0:1788574612.886148:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574612.886154:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574612.886159:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306217401
00000100:00000010:0.0:1788574612.886161:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d74e0.
00000100:00000001:0.0:1788574612.886163:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574612.886166:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574612.886168:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.886170:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574612.886173:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574612/real 1788574612]  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574612.886180:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574612.886181:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574612.886183:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574612.886185:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574612.886186:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574612.886188:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574612.886190:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574612.886196:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886200:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574612.886201:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886203:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574612.886204:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574612.886207:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886211:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574612.886214:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886219:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574612.886221:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886222:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574612.886223:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886224:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574612.886226:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574612.886228:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574612.886231:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574612.886233:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574612.886236:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574612.886241:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393147520:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574612.886245:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886246:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574612.886248:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4380 x1875444393147520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886252:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574612.886254:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574612.886255:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574612.886257:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574612.886259:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574612.886260:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574612.886261:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574612.886263:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:0.0:1788574612.886265:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574612.886266:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574612.886268:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4380.
00000100:00000001:0.0:1788574612.886269:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574612.886270:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886272:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886278:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574612.886279:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.886281:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574612.886282:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574612.886283:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574612.886285:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886290:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574612.886291:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574612.886293:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574612.886294:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574612.886296:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574612.886298:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574612.886300:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574612.886301:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574612.886302:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574612.886304:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574612.886306:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574612.886307:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574612.886310:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574612.886312:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574612.886314:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574612.886316:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393147648, portal 10
00000100:00000001:0.0:1788574612.886318:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574612.886319:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345465856 : -130483364085760 : ffff8953796f4000)
00000100:00000040:0.0:1788574612.886323:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886328:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574612.886329:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574612.886330:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574612.886332:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393147648, offset 0
00000400:00000200:0.0:1788574612.886334:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574612.886337:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574612.886340:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574612.886342:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574612.886344:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574612.886345:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574612.886347:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574612.886349:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330409:330409:256:4294967295] 192.168.202.110@tcp LPNI seq info [330366:330366:8:4294967295]
00000400:00000200:0.0:1788574612.886352:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574612.886356:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574612.886360:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574612.886362:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574612.886365:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574612.886367:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788574612.886370:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574612.886372:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1788574612.886374:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574612.886376:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574612.886378:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574612.886380:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574612.886382:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574612.886384:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574612.886385:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574612.886388:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886418:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574612.886421:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574637 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886425:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.886426:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574612.886429:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574612.886431:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574612.886434:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574612.886435:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306216487
00000100:00000001:0.0:1788574612.886438:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306217401
00000100:00000001:0.0:1788574612.886439:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306216487
00000100:00000001:0.0:1788574612.886441:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306217401
00000100:00000001:0.0:1788574612.886442:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574612.886443:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.886445:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574612.886447:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574612/real 1788574612]  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574612.886453:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574612.886454:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574612.886455:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574612.886457:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574612.886458:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574612.886459:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574612.886461:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574612.886466:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886470:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574612.886471:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886472:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574612.886473:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574612.886476:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886480:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574612.886482:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886486:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574612.886488:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886489:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574612.886489:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886490:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574612.886492:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574612.886493:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574612.886495:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574612.886497:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574612.886500:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574612.886505:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574612.886508:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574612.886510:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886514:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574612.886515:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574612.886516:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574612.886517:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574612.886518:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574612.886520:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574612.886522:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574612.886523:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574612.886524:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574612.886525:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574612.886527:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574612.886529:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574612.886531:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574612.886533:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574612.886535:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574612.886537:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393147776, portal 4
00000100:00000001:0.0:1788574612.886538:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574612.886539:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345473920 : -130483364077696 : ffff8953796f5f80)
00000100:00000040:0.0:1788574612.886543:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886547:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574612.886549:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574612.886551:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574612.886552:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393147776, offset 0
00000400:00000200:0.0:1788574612.886555:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574612.886558:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574612.886560:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574612.886562:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574612.886564:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574612.886565:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574612.886567:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574612.886569:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330410:330410:256:4294967295] 192.168.202.110@tcp LPNI seq info [330367:330367:8:4294967295]
00000400:00000200:0.0:1788574612.886572:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574612.886576:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574612.886578:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574612.886580:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574612.886583:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574612.886585:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1788574612.886587:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574612.886589:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1788574612.886591:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574612.886593:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574612.886595:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574612.886597:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574612.886599:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574612.886600:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574612.886601:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574612.886604:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886608:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574612.886610:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574637 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886615:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.886616:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574612.886617:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574612.886619:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574612.886621:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574612.886623:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306216633
00000100:00000001:0.0:1788574612.886624:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306217401
00000100:00000001:0.0:1788574612.886626:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306216633
00000100:00000001:0.0:1788574612.886627:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306217401
00000100:00000001:0.0:1788574612.886628:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574612.886630:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.886631:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574612.886633:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574612/real 1788574612]  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574612.886640:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574612.886642:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574612.886643:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574612.886645:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574612.886647:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574612.886649:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574612.886652:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574612.886661:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886668:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574612.886669:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886671:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574612.886673:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574612.886677:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886685:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574612.886690:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886696:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574612.886698:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886699:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574612.886700:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886701:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574612.886703:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574612.886704:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574612.886706:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574612.886708:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574612.886711:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574612.886716:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393147776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574612.886719:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886720:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574612.886722:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4000 x1875444393147648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886726:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574612.886727:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574612.886728:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574612.886730:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574612.886731:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574612.886732:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574612.886733:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574612.886735:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:0.0:1788574612.886737:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574612.886738:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574612.886739:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4000.
00000100:00000001:0.0:1788574612.886741:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574612.886741:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886743:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574612.886745:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5f80 x1875444393147776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574637 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574612.886753:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574612.886753:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574612.886755:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574612.886756:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574612.886757:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574612.886757:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574612.886758:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574612.886759:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:0.0:1788574612.886761:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574612.886761:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574612.886762:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5f80.
00000100:00000001:0.0:1788574612.886764:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574612.886764:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886765:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574612.886768:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574612.886769:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574612.886770:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574612.886771:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574613.845512:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306217401, at 4306217408
00000100:00000040:1.0:1788574613.845512:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306217401, at 4306217408
00000100:00000010:1.0:1788574613.845528:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d74e0.
00000100:00000010:0.0:1788574613.845530:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574618.006364:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574618.006369:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574618.006374:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574618.006385:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.006436:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574618.006440:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574618.006445:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574618.006449:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574618.006453:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574618.006461:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393147904:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574618.006469:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574618.006476:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574618.006482:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574618.006486:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574618.006492:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574618.006495:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574618.006504:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574618.006510:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574618.006520:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574618.006529:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393147904, portal 25
00000100:00000001:0.0:1788574618.006533:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574618.006536:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:0.0:1788574618.006547:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.006561:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574618.006567:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574618.006572:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574618.006578:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393147904, offset 0
00000400:00000200:0.0:1788574618.006589:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574618.006602:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574618.006611:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574618.006620:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574618.006626:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574618.006632:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574618.006637:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574618.006644:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330411:330411:256:4294967295] 192.168.202.110@tcp LPNI seq info [330368:330368:8:4294967295]
00000400:00000200:0.0:1788574618.006654:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574618.006667:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574618.006677:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574618.006684:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574618.006705:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574618.006710:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788574618.006722:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574618.006727:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788574618.006735:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574618.006741:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574618.006750:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574618.006758:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574618.006765:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574618.006769:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574618.006777:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574618.006781:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.006789:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574618.006793:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574643 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.006799:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574618.006801:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574618.006807:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574618.006810:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574618.006812:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306222522
00000100:00000010:0.0:1788574618.006816:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574618.006823:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574618.006830:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306222522
00000100:00000010:0.0:1788574618.006832:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7420.
00000100:00000001:0.0:1788574618.006835:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574618.006837:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574618.006840:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574618.006842:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574618.006845:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574618/real 1788574618]  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574618.006853:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574618.006855:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574618.006857:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574618.006858:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574618.006860:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574618.006862:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574618.006865:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574618.006871:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.006876:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574618.006877:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.006879:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574618.006882:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574618.006885:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.006889:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574618.006892:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.006897:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574618.006899:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.006900:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574618.006901:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.006903:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574618.006904:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574618.006906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574618.006909:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574618.006911:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574618.006914:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574618.006920:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393147904:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574618.006925:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.006926:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574618.006928:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393147904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.006937:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574618.006939:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574618.006942:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574618.006944:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574618.006946:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574618.006947:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574618.006948:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574618.006950:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788574618.006953:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574618.006955:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574618.006956:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:0.0:1788574618.006959:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574618.006960:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.006962:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.006969:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574618.006971:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574618.006973:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574618.006974:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574618.006975:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574618.006977:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.006982:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574618.006983:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574618.006985:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574618.006986:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574618.006988:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574618.006989:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148032:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574618.006991:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574618.006992:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574618.006993:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574618.006995:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574618.006997:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574618.006998:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574618.007001:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574618.007003:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574618.007006:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574618.007008:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393148032, portal 10
00000100:00000001:0.0:1788574618.007009:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574618.007010:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:0.0:1788574618.007015:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007020:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574618.007021:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574618.007023:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574618.007025:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393148032, offset 0
00000400:00000200:0.0:1788574618.007027:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574618.007029:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574618.007031:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574618.007033:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574618.007035:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574618.007037:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574618.007038:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574618.007041:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330412:330412:256:4294967295] 192.168.202.110@tcp LPNI seq info [330369:330369:8:4294967295]
00000400:00000200:0.0:1788574618.007044:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574618.007066:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574618.007070:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574618.007072:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574618.007075:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574618.007077:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788574618.007080:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574618.007082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788574618.007085:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574618.007087:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574618.007089:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574618.007091:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574618.007094:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574618.007095:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574618.007097:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574618.007100:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007105:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574618.007107:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574643 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007112:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574618.007113:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574618.007115:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574618.007119:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574618.007121:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574618.007123:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306221607
00000100:00000001:0.0:1788574618.007125:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306222522
00000100:00000001:0.0:1788574618.007127:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306221607
00000100:00000001:0.0:1788574618.007128:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306222522
00000100:00000001:0.0:1788574618.007130:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574618.007131:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574618.007132:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574618.007134:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574618/real 1788574618]  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574618.007146:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574618.007147:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574618.007149:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574618.007150:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574618.007152:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574618.007153:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574618.007156:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574618.007161:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007170:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574618.007171:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007173:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574618.007175:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574618.007179:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007187:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574618.007192:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007200:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574618.007203:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007205:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574618.007206:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007209:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574618.007211:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574618.007214:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574618.007217:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574618.007219:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574618.007222:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574618.007228:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148032:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574618.007230:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574618.007232:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007237:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574618.007238:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574618.007240:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574618.007242:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574618.007243:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574618.007245:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148160:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574618.007247:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574618.007249:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574618.007250:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574618.007251:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574618.007254:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574618.007255:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574618.007258:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574618.007260:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574618.007263:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574618.007265:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393148160, portal 4
00000100:00000001:0.0:1788574618.007267:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574618.007267:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:0.0:1788574618.007272:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007277:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574618.007278:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574618.007280:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574618.007281:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393148160, offset 0
00000400:00000200:0.0:1788574618.007284:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574618.007287:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574618.007289:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574618.007291:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574618.007292:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574618.007294:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574618.007296:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574618.007298:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330413:330413:256:4294967295] 192.168.202.110@tcp LPNI seq info [330370:330370:8:4294967295]
00000400:00000200:0.0:1788574618.007301:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574618.007306:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574618.007310:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574618.007312:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574618.007315:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574618.007317:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788574618.007320:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574618.007321:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788574618.007324:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574618.007326:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574618.007328:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574618.007330:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574618.007332:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574618.007334:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574618.007335:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574618.007338:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574643 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007343:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574618.007345:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574643 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007349:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574618.007351:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574618.007353:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574618.007355:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574618.007357:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574618.007359:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306221753
00000100:00000001:0.0:1788574618.007361:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306222522
00000100:00000001:0.0:1788574618.007362:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306221753
00000100:00000001:0.0:1788574618.007364:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306222522
00000100:00000001:0.0:1788574618.007365:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574618.007366:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574618.007368:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574618.007370:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574618/real 1788574618]  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574618.007376:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574618.007377:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574618.007378:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574618.007380:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574618.007381:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574618.007382:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574618.007384:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574618.007389:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007409:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574618.007411:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007412:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574618.007413:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574618.007416:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007421:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574618.007423:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007428:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574618.007429:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007430:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574618.007431:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007432:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574618.007433:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574618.007435:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574618.007436:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574618.007438:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574618.007441:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574618.007447:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148160:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574618.007449:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007450:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574618.007452:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393148032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007456:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574618.007457:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574618.007458:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574618.007460:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574618.007461:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574618.007462:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574618.007464:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574618.007465:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788574618.007467:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574618.007468:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574618.007469:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:0.0:1788574618.007471:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574618.007472:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007473:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574618.007475:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393148160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574643 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574618.007480:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574618.007480:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574618.007481:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574618.007482:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574618.007483:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574618.007484:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574618.007485:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574618.007486:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:0.0:1788574618.007487:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574618.007488:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574618.007489:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:0.0:1788574618.007491:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574618.007492:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007494:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574618.007496:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574618.007497:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574618.007499:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574618.007500:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574618.965497:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306222522, at 4306222528
00000100:00000040:1.0:1788574618.965497:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306222522, at 4306222528
00000100:00000010:1.0:1788574618.965514:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7420.
00000100:00000010:0.0:1788574618.965515:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574623.125685:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574623.125691:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574623.125696:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574623.125708:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.125727:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574623.125731:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574623.125737:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574623.125742:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574623.125746:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574623.125750:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393148288:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574623.125768:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574623.125775:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574623.125780:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574623.125786:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574623.125794:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574623.125800:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574623.125812:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574623.125820:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574623.125831:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574623.125842:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393148288, portal 25
00000100:00000001:0.0:1788574623.125848:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574623.125853:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:0.0:1788574623.125866:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574648 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.125891:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574623.125902:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574623.125908:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574623.125915:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393148288, offset 0
00000400:00000200:0.0:1788574623.125928:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574623.125952:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574623.125964:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574623.125973:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574623.125981:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574623.125989:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574623.125996:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574623.126019:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330414:330414:256:4294967295] 192.168.202.110@tcp LPNI seq info [330371:330371:8:4294967295]
00000400:00000200:0.0:1788574623.126025:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574623.126039:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574623.126047:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574623.126053:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574623.126061:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574623.126065:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574623.126071:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574623.126075:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574623.126080:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574623.126082:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574623.126096:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574623.126102:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574623.126108:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574623.126111:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574623.126116:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574623.126120:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574648 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126130:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574623.126134:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574648 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126147:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574623.126149:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574623.126155:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574623.126159:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574623.126162:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306227641
00000100:00000010:0.0:1788574623.126167:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574623.126174:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574623.126180:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306227641
00000100:00000010:0.0:1788574623.126183:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d72a0.
00000100:00000001:0.0:1788574623.126186:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574623.126190:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574623.126192:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574623.126195:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574623.126199:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574623/real 1788574623]  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574623.126209:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574623.126211:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574623.126213:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574623.126217:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574623.126219:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574623.126221:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574623.126225:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574623.126234:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126240:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574623.126243:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126246:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574623.126248:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574623.126252:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126259:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574623.126263:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126271:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574623.126274:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126275:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574623.126277:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126279:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574623.126281:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574623.126284:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574623.126287:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574623.126290:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574623.126295:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574623.126302:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393148288:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574623.126307:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126309:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574623.126312:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393148288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126319:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574623.126321:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574623.126324:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574623.126326:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574623.126328:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574623.126330:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574623.126332:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574623.126335:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788574623.126338:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574623.126341:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574623.126343:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:0.0:1788574623.126346:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574623.126347:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126349:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126356:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574623.126358:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574623.126360:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574623.126361:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574623.126363:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574623.126366:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126373:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574623.126374:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574623.126376:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574623.126378:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574623.126380:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574623.126383:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148416:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574623.126387:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574623.126390:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574623.126426:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574623.126429:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574623.126432:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574623.126434:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574623.126438:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574623.126441:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574623.126445:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574623.126448:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393148416, portal 10
00000100:00000001:0.0:1788574623.126450:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574623.126451:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:0.0:1788574623.126458:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574648 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126464:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574623.126467:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574623.126469:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574623.126472:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393148416, offset 0
00000400:00000200:0.0:1788574623.126476:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574623.126480:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574623.126483:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574623.126487:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574623.126489:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574623.126492:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574623.126494:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574623.126497:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330415:330415:256:4294967295] 192.168.202.110@tcp LPNI seq info [330372:330372:8:4294967295]
00000400:00000200:0.0:1788574623.126501:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574623.126507:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574623.126512:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574623.126515:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574623.126519:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574623.126521:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574623.126526:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574623.126529:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574623.126533:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574623.126535:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574623.126538:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574623.126541:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574623.126544:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574623.126546:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574623.126549:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574623.126552:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574648 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126560:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574623.126563:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574648 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126570:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574623.126571:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574623.126585:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574623.126589:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574623.126592:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574623.126595:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306226727
00000100:00000001:0.0:1788574623.126597:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306227641
00000100:00000001:0.0:1788574623.126600:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306226727
00000100:00000001:0.0:1788574623.126602:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306227641
00000100:00000001:0.0:1788574623.126604:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574623.126606:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574623.126609:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574623.126613:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574623/real 1788574623]  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574623.126622:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574623.126623:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574623.126625:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574623.126628:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574623.126629:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574623.126632:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574623.126634:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574623.126642:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126649:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574623.126651:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126653:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574623.126654:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574623.126657:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126664:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574623.126668:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126676:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574623.126678:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126680:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574623.126681:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126683:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574623.126685:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574623.126687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574623.126691:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574623.126693:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574623.126697:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574623.126705:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148416:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574623.126709:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574623.126712:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126719:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574623.126720:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574623.126722:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574623.126724:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574623.126726:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574623.126729:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574623.126732:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574623.126734:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574623.126736:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574623.126737:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574623.126741:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574623.126743:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574623.126747:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574623.126750:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574623.126753:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574623.126756:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393148544, portal 4
00000100:00000001:0.0:1788574623.126758:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574623.126760:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:0.0:1788574623.126765:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574648 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126771:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574623.126774:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574623.126776:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574623.126778:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393148544, offset 0
00000400:00000200:0.0:1788574623.126782:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574623.126786:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574623.126790:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574623.126793:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574623.126796:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574623.126798:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574623.126800:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574623.126804:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330416:330416:256:4294967295] 192.168.202.110@tcp LPNI seq info [330373:330373:8:4294967295]
00000400:00000200:0.0:1788574623.126808:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574623.126815:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574623.126819:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574623.126823:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574623.126828:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574623.126831:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788574623.126835:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574623.126838:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788574623.126841:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574623.126843:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574623.126846:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574623.126849:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574623.126852:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574623.126854:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574623.126857:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574623.126861:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574648 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126868:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574623.126872:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574648 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126879:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574623.126880:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574623.126884:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574623.126887:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574623.126890:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574623.126892:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306226873
00000100:00000001:0.0:1788574623.126895:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306227641
00000100:00000001:0.0:1788574623.126898:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306226873
00000100:00000001:0.0:1788574623.126900:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306227641
00000100:00000001:0.0:1788574623.126902:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574623.126904:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574623.126906:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574623.126911:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574623/real 1788574623]  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574623.126919:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574623.126921:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574623.126922:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574623.126925:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574623.126926:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574623.126928:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574623.126932:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574623.126942:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126954:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574623.126956:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126958:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574623.126959:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574623.126963:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126970:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574623.126974:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.126981:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574623.126983:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126985:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574623.126986:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.126989:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574623.126990:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574623.126993:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574623.126996:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574623.126999:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574623.127022:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574623.127031:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574623.127035:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.127037:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574623.127041:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393148416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.127047:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574623.127049:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574623.127052:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574623.127054:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574623.127056:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574623.127058:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574623.127060:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574623.127062:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:0.0:1788574623.127064:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574623.127066:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574623.127068:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:0.0:1788574623.127071:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574623.127072:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.127074:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574623.127078:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393148544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574648 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574623.127085:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574623.127086:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574623.127088:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574623.127090:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574623.127091:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574623.127093:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574623.127095:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574623.127096:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788574623.127098:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574623.127100:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574623.127101:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:0.0:1788574623.127103:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574623.127105:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.127107:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574623.127110:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574623.127112:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574623.127114:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574623.127115:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574624.085493:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306227641, at 4306227648
00000100:00000040:1.0:1788574624.085493:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306227641, at 4306227648
00000100:00000010:0.0:1788574624.085509:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574624.085512:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d72a0.
00000100:00080000:0.0:1788574628.245479:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574628.245489:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574628.245492:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574628.245495:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574628.245499:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574628.245504:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11559
00000400:00000200:0.0:1788574628.245512:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574628.245517:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574628.245519:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574628.245522:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574628.245525:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574628.245528:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574628.245530:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574628.245535:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574628.245536:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574628.245540:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574628.245543:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574628.245546:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574628.245548:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574628.245552:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574628.245554:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574628.245556:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574628.245558:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574628.245561:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574628.245565:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574628.245568:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574628.245571:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574628.245576:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574628.245579:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574628.245581:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574628.245583:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574628.245591:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788574628.245594:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574628.245606:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574628.245608:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574628.245611:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574628.245614:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574628.245619:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574628.245627:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574628.245635:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574628.245644:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574628.245654:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574628.245657:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574628.245662:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574628.245681:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574628.245683:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574628.245686:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574628.245687:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574628.245688:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574628.245691:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574628.245695:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11559
00000400:00000200:0.0:1788574628.245700:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574628.245704:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1788574628.245705:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1788574628.245707:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1788574628.245710:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1788574628.245714:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574628.245715:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788574628.245720:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574628.245724:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574628.245727:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574628.245733:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:1.0:1788574628.245739:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1788574628.245740:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1788574628.245742:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788574628.245743:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574628.245747:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:1.0:1788574628.245748:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1788574628.245749:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
02000000:00000001:1.0:1788574628.245752:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1788574628.245753:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574628.245758:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:1.0:1788574628.245758:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574628.245766:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393148672:192.168.202.110@tcp:250:kworker.0
00000100:00000040:0.0:1788574628.245767:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574628.245771:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574628.245775:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574628.245775:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1788574628.245778:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:1.0:1788574628.245779:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:0.0:1788574628.245780:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:1.0:1788574628.245783:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574628.245786:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1788574628.245787:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788574628.245791:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574628.245794:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:1.0:1788574628.245795:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574628.245798:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574628.245799:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574628.245802:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574628.245806:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574628.245807:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1788574628.245812:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000008:00100000:0.0:1788574628.245813:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574628.245817:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574628.245819:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1788574628.245821:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000010:0.0:1788574628.245823:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574628.245826:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574628.245828:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:1.0:1788574628.245828:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393148672, portal 25
02000000:00000001:0.0:1788574628.245832:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.245833:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574628.245836:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1788574628.245836:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
02000000:00000001:0.0:1788574628.245839:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574628.245842:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574628.245847:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788574628.245850:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574628.245854:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574653 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574628.245856:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574628.245874:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788574628.245877:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1788574628.245879:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788574628.245881:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1788574628.245884:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574628.245889:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393148672, offset 0
00000100:00000040:0.0:1788574628.245890:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574628.245899:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574628.245911:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574628.245920:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788574628.245922:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574628.245925:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1788574628.245928:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1788574628.245932:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1788574628.245936:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:1.0:1788574628.245938:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788574628.245940:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1788574628.245943:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:0.0:1788574628.245944:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1788574628.245961:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330417:330417:256:4294967295] 192.168.202.110@tcp LPNI seq info [330374:330374:8:4294967295]
00000100:00000001:0.0:1788574628.245963:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:1.0:1788574628.245967:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788574628.245970:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11559
00000400:00000200:1.0:1788574628.245979:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1788574628.245986:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.0:1788574628.245987:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574628.245993:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:1.0:1788574628.245994:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000100:00080000:0.0:1788574628.245997:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574628.246001:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:1.0:1788574628.246005:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574628.246006:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1788574628.246011:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:0.0:1788574628.246012:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574628.246017:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574628.246023:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:1.1:1788574628.246023:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1788574628.246026:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:1.1:1788574628.246028:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1788574628.246031:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000100:1.0:1788574628.246034:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:0.0:1788574628.246037:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000010:1.0:1788574628.246037:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000020:00000001:0.0:1788574628.246043:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:1.0:1788574628.246044:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1788574628.246050:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:1.0:1788574628.246052:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574628.246058:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000100:00000040:0.0:1788574628.246060:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:1.0:1788574628.246061:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:0.0:1788574628.246066:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1788574628.246068:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:0.0:1788574628.246072:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:1.0:1788574628.246074:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574653 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574628.246078:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574628.246084:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574628.246090:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574628.246091:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1788574628.246095:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574628.246097:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574653 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788574628.246098:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574628.246102:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574628.246107:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574628.246109:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574628.246109:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246111:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000008:00100000:0.0:1788574628.246113:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574628.246116:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574628.246118:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246118:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574628.246126:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1788574628.246127:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000100:00000001:1.0:1788574628.246129:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306232761
00000020:00000040:0.0:1788574628.246133:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000010:1.0:1788574628.246137:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9e40.
02000000:00000001:0.0:1788574628.246139:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574628.246142:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246144:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574628.246145:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574628.246150:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1788574628.246152:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306232761
02000000:00000001:0.0:1788574628.246153:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574628.246156:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9960.
00000100:00000001:1.0:1788574628.246160:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574628.246161:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:0.0:1788574628.246162:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:1.0:1788574628.246163:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246168:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1788574628.246171:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574628.246173:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574628/real 1788574628]  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574628.246182:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:1.0:1788574628.246193:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574628.246196:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1788574628.246201:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1788574628.246201:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574628.246206:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000100:00000001:0.0:1788574628.246207:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1788574628.246208:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574628.246212:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000040:0.0:1788574628.246218:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574628.246218:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574628.246234:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574628.246239:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574628.246244:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000200:1.0:1788574628.246246:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574628.246249:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11569)
00000100:00000001:1.0:1788574628.246256:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574628.246258:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246260:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574628.246262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574628.246265:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246270:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574628.246274:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246279:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574628.246280:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246282:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574628.246283:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246284:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574628.246286:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574628.246288:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574628.246291:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574628.246292:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574628.246295:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574628.246301:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393148672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574628.246305:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246306:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574628.246308:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393148672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246312:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574628.246314:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574628.246316:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574628.246318:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574628.246320:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574628.246321:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574628.246322:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574628.246324:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:1.0:1788574628.246326:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574628.246328:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574628.246329:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:1.0:1788574628.246331:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574628.246332:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246334:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246340:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574628.246341:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246343:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574628.246344:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574628.246345:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574628.246347:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246352:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574628.246353:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574628.246355:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574628.246356:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574628.246357:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574628.246359:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574628.246362:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574628.246363:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574628.246365:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574628.246366:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574628.246370:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574628.246372:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574628.246375:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574628.246378:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574628.246382:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574628.246386:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393148800, portal 10
00000100:00000001:1.0:1788574628.246388:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574628.246389:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:1.0:1788574628.246419:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574653 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246424:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574628.246425:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574628.246428:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574628.246430:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393148800, offset 0
00000400:00000200:1.0:1788574628.246433:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574628.246436:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574628.246439:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574628.246441:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574628.246443:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574628.246445:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574628.246447:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574628.246449:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330418:330418:256:4294967295] 192.168.202.110@tcp LPNI seq info [330375:330375:8:4294967295]
00000400:00000200:1.0:1788574628.246452:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574628.246456:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574628.246459:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574628.246461:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574628.246463:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574628.246466:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1788574628.246469:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574628.246470:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1788574628.246473:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574628.246474:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574628.246477:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574628.246479:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574628.246481:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574628.246482:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574628.246484:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574628.246486:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574653 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246492:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574628.246494:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574653 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246498:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246499:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574628.246501:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574628.246504:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574628.246506:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574628.246508:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306231847
00000100:00000001:1.0:1788574628.246510:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306232761
00000100:00000001:1.0:1788574628.246511:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306231847
00000100:00000001:1.0:1788574628.246513:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306232761
00000100:00000001:1.0:1788574628.246514:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574628.246515:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246517:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574628.246519:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574628/real 1788574628]  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574628.246524:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574628.246525:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574628.246526:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574628.246528:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574628.246529:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574628.246531:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574628.246534:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574628.246543:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246550:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574628.246552:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246553:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574628.246555:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574628.246559:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246566:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574628.246571:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246579:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574628.246582:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246584:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574628.246585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246587:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574628.246589:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574628.246592:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574628.246594:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574628.246596:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574628.246599:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574628.246604:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574628.246607:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574628.246610:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246614:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574628.246615:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574628.246617:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574628.246618:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574628.246619:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574628.246621:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148928:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574628.246623:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574628.246625:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574628.246626:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574628.246627:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574628.246630:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574628.246631:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574628.246634:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574628.246636:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574628.246638:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574628.246640:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393148928, portal 4
00000100:00000001:1.0:1788574628.246642:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574628.246642:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:1.0:1788574628.246646:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574653 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246651:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574628.246653:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574628.246654:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574628.246655:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393148928, offset 0
00000400:00000200:1.0:1788574628.246658:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574628.246661:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574628.246663:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574628.246665:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574628.246667:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574628.246669:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574628.246671:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574628.246673:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330419:330419:256:4294967295] 192.168.202.110@tcp LPNI seq info [330376:330376:8:4294967295]
00000400:00000200:1.0:1788574628.246675:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574628.246680:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574628.246682:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574628.246684:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574628.246688:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574628.246691:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1788574628.246695:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574628.246698:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1788574628.246701:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574628.246703:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574628.246706:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574628.246709:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574628.246712:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574628.246715:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574628.246717:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574628.246721:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574653 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246727:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574628.246729:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574653 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246733:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246734:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574628.246737:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574628.246739:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574628.246741:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574628.246743:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306231993
00000100:00000001:1.0:1788574628.246744:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306232761
00000100:00000001:1.0:1788574628.246746:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306231993
00000100:00000001:1.0:1788574628.246748:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306232761
00000100:00000001:1.0:1788574628.246749:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574628.246751:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246752:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574628.246754:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574628/real 1788574628]  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574628.246759:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574628.246760:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574628.246761:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574628.246763:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574628.246764:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574628.246765:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574628.246767:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574628.246772:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246776:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574628.246777:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246778:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574628.246779:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574628.246781:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246785:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574628.246788:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246792:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574628.246793:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246794:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574628.246795:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246796:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574628.246798:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574628.246799:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574628.246802:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574628.246803:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574628.246805:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574628.246810:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393148928:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574628.246812:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246813:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574628.246815:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393148800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246819:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574628.246821:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574628.246822:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574628.246824:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574628.246825:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574628.246826:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574628.246827:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574628.246829:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:1.0:1788574628.246830:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574628.246831:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574628.246832:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:1.0:1788574628.246834:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574628.246835:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246836:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574628.246838:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393148928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574653 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574628.246842:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574628.246843:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574628.246844:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574628.246846:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574628.246847:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574628.246848:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574628.246849:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574628.246850:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:1.0:1788574628.246851:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574628.246852:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574628.246853:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:1.0:1788574628.246854:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574628.246855:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246856:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574628.246858:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574628.246859:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574628.246860:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574628.246862:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574629.205517:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306232761, at 4306232768
00000100:00000040:1.0:1788574629.205517:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306232761, at 4306232768
00000100:00000010:0.0:1788574629.205533:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9e40.
00000100:00000010:1.0:1788574629.205536:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9960.
00000100:00000001:1.0:1788574633.365771:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574633.365777:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574633.365781:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574633.365791:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.365800:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574633.365804:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574633.365808:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574633.365811:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574633.365815:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574633.365819:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393149056:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574633.365826:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574633.365831:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574633.365834:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574633.365838:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574633.365844:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574633.365848:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574633.365855:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574633.365927:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574633.365937:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574633.365944:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393149056, portal 25
00000100:00000001:1.0:1788574633.365947:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574633.365951:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:1.0:1788574633.365959:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.365978:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574633.365984:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574633.365989:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574633.365994:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393149056, offset 0
00000400:00000200:1.0:1788574633.366001:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574633.366012:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574633.366020:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574633.366031:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574633.366041:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574633.366050:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574633.366057:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574633.366066:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330420:330420:256:4294967295] 192.168.202.110@tcp LPNI seq info [330377:330377:8:4294967295]
00000400:00000200:1.0:1788574633.366079:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574633.366099:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574633.366111:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574633.366121:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574633.366134:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574633.366140:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1788574633.366149:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574633.366154:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1788574633.366163:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574633.366167:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574633.366176:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574633.366183:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574633.366191:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574633.366194:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574633.366201:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574633.366208:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366224:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574633.366229:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366237:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574633.366239:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574633.366244:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574633.366248:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574633.366250:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306237881
00000100:00000010:1.0:1788574633.366254:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9960.
00000100:00000001:1.0:1788574633.366260:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574633.366265:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306237881
00000100:00000010:1.0:1788574633.366267:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9b40.
00000100:00000001:1.0:1788574633.366269:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574633.366271:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574633.366273:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574633.366275:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574633.366278:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574633/real 1788574633]  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574633.366285:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574633.366286:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574633.366288:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574633.366290:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574633.366291:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574633.366292:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574633.366295:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574633.366303:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366316:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574633.366318:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366321:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574633.366323:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574633.366328:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366336:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574633.366341:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366350:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574633.366353:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366354:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574633.366356:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366358:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574633.366360:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574633.366363:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574633.366367:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574633.366370:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574633.366375:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574633.366422:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393149056:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574633.366427:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366429:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574633.366431:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393149056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366436:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574633.366438:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574633.366441:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574633.366443:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574633.366445:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574633.366446:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574633.366448:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574633.366450:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:1.0:1788574633.366452:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574633.366454:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574633.366455:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:1.0:1788574633.366457:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574633.366458:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366460:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366466:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574633.366467:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574633.366469:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574633.366470:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574633.366471:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574633.366473:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366485:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574633.366486:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574633.366487:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574633.366489:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574633.366490:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574633.366493:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149184:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574633.366495:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574633.366496:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574633.366497:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574633.366498:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574633.366501:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574633.366502:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574633.366505:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574633.366507:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574633.366509:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574633.366512:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393149184, portal 10
00000100:00000001:1.0:1788574633.366514:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574633.366515:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:1.0:1788574633.366519:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366524:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574633.366525:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574633.366527:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574633.366528:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393149184, offset 0
00000400:00000200:1.0:1788574633.366531:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574633.366534:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574633.366537:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574633.366539:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574633.366541:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574633.366542:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574633.366544:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574633.366546:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330421:330421:256:4294967295] 192.168.202.110@tcp LPNI seq info [330378:330378:8:4294967295]
00000400:00000200:1.0:1788574633.366549:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574633.366553:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574633.366557:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574633.366559:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574633.366562:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574633.366564:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1788574633.366567:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574633.366569:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1788574633.366571:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574633.366572:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574633.366574:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574633.366577:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574633.366579:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574633.366580:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574633.366582:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574633.366584:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366594:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574633.366596:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366600:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574633.366601:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574633.366603:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574633.366605:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574633.366607:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574633.366609:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306236967
00000100:00000001:1.0:1788574633.366611:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306237881
00000100:00000001:1.0:1788574633.366612:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306236967
00000100:00000001:1.0:1788574633.366614:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306237881
00000100:00000001:1.0:1788574633.366615:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574633.366617:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574633.366618:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574633.366620:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574633/real 1788574633]  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574633.366625:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574633.366626:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574633.366627:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574633.366628:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574633.366630:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574633.366631:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574633.366633:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574633.366638:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366642:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574633.366643:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366644:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574633.366646:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574633.366648:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366652:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574633.366655:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366659:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574633.366660:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366661:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574633.366662:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366663:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574633.366665:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574633.366666:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574633.366668:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574633.366670:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574633.366672:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574633.366677:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149184:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574633.366679:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574633.366681:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366685:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574633.366686:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574633.366688:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574633.366689:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574633.366690:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574633.366692:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149312:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574633.366694:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574633.366695:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574633.366697:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574633.366698:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574633.366700:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574633.366701:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574633.366703:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574633.366705:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574633.366707:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574633.366709:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393149312, portal 4
00000100:00000001:1.0:1788574633.366710:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574633.366711:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:1.0:1788574633.366715:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366719:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574633.366720:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574633.366721:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574633.366723:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393149312, offset 0
00000400:00000200:1.0:1788574633.366726:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574633.366728:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574633.366730:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574633.366731:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574633.366733:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574633.366735:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574633.366736:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574633.366738:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330422:330422:256:4294967295] 192.168.202.110@tcp LPNI seq info [330379:330379:8:4294967295]
00000400:00000200:1.0:1788574633.366741:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574633.366745:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574633.366748:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574633.366749:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574633.366752:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574633.366754:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1788574633.366757:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574633.366759:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1788574633.366761:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574633.366762:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574633.366764:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574633.366766:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574633.366768:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574633.366769:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574633.366771:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574633.366773:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366777:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574633.366779:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366784:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574633.366785:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574633.366786:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574633.366788:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574633.366790:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574633.366791:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306237113
00000100:00000001:1.0:1788574633.366792:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306237881
00000100:00000001:1.0:1788574633.366794:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306237113
00000100:00000001:1.0:1788574633.366795:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306237881
00000100:00000001:1.0:1788574633.366797:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574633.366798:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574633.366799:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574633.366802:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574633/real 1788574633]  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574633.366806:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574633.366807:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574633.366808:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574633.366810:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574633.366811:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574633.366812:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574633.366814:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574633.366819:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366823:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574633.366823:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366824:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574633.366825:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574633.366827:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366831:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574633.366834:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366837:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574633.366839:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366840:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574633.366841:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366842:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574633.366843:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574633.366844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574633.366846:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574633.366848:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574633.366850:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574633.366860:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149312:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574633.366862:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366864:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574633.366865:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393149184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366869:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574633.366870:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574633.366871:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574633.366873:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574633.366873:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574633.366874:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574633.366876:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574633.366877:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:1.0:1788574633.366878:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574633.366879:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574633.366881:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:1.0:1788574633.366882:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574633.366883:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366884:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574633.366886:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393149312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574658 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574633.366890:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574633.366891:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574633.366892:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574633.366893:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574633.366894:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574633.366895:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574633.366896:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574633.366912:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:1.0:1788574633.366916:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574633.366917:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574633.366918:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:1.0:1788574633.366921:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574633.366922:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366924:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574633.366926:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574633.366927:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574633.366928:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574633.366929:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574634.325482:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306237881, at 4306237888
00000100:00000040:1.0:1788574634.325482:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306237881, at 4306237888
00000100:00000010:1.0:1788574634.325496:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9b40.
00000100:00000010:0.0:1788574634.325499:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9960.
00000800:00000200:0.0:1788574636.437795:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574636.437799:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574636.437818:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574636.437823:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574636.437825:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.485713:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574638.485721:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574638.485726:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574638.485742:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.485781:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574638.485785:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574638.485792:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574638.485800:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574638.485810:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574638.485831:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393149440:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574638.485868:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574638.485875:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574638.485880:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574638.485888:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574638.485902:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574638.485907:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574638.485925:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574638.485932:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574638.485946:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574638.485957:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393149440, portal 25
00000100:00000001:1.0:1788574638.485968:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574638.485977:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:1.0:1788574638.485996:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486026:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574638.486039:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574638.486047:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574638.486056:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393149440, offset 0
00000400:00000200:1.0:1788574638.486069:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574638.486092:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574638.486105:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574638.486120:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574638.486127:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574638.486134:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574638.486139:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574638.486151:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330423:330423:256:4294967295] 192.168.202.110@tcp LPNI seq info [330380:330380:8:4294967295]
00000400:00000200:1.0:1788574638.486178:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574638.486215:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574638.486240:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574638.486255:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574638.486263:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574638.486267:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1788574638.486275:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574638.486279:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1788574638.486283:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574638.486285:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574638.486293:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574638.486297:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574638.486303:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574638.486305:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574638.486310:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574638.486314:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486323:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574638.486327:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486332:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574638.486334:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574638.486340:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574638.486343:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574638.486345:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306243001
00000100:00000010:1.0:1788574638.486351:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9b40.
00000100:00000001:1.0:1788574638.486359:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574638.486374:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306243001
00000100:00000010:1.0:1788574638.486377:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9240.
00000100:00000001:1.0:1788574638.486380:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574638.486383:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574638.486385:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574638.486387:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574638.486432:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574638/real 1788574638]  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574638.486443:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574638.486445:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574638.486448:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574638.486450:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574638.486451:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574638.486453:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574638.486457:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574638.486464:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486479:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574638.486481:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486485:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574638.486487:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574638.486490:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486496:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574638.486500:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486504:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574638.486507:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486508:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574638.486509:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486511:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574638.486513:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574638.486515:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574638.486518:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574638.486520:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574638.486523:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574638.486528:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393149440:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574638.486532:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486534:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574638.486537:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393149440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486541:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574638.486543:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574638.486546:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574638.486548:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574638.486550:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574638.486552:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574638.486553:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574638.486556:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:1.0:1788574638.486560:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574638.486562:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574638.486564:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:1.0:1788574638.486567:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574638.486568:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486570:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486577:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574638.486579:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574638.486581:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574638.486582:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574638.486583:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574638.486586:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486592:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574638.486593:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574638.486595:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574638.486597:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574638.486598:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574638.486600:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574638.486602:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574638.486604:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574638.486605:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574638.486606:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574638.486609:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574638.486610:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574638.486613:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574638.486615:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574638.486618:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574638.486620:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393149568, portal 10
00000100:00000001:1.0:1788574638.486622:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574638.486623:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000040:1.0:1788574638.486628:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486633:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574638.486635:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574638.486637:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574638.486638:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393149568, offset 0
00000400:00000200:1.0:1788574638.486641:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574638.486644:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574638.486647:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574638.486649:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574638.486651:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574638.486653:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574638.486655:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574638.486657:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330424:330424:256:4294967295] 192.168.202.110@tcp LPNI seq info [330381:330381:8:4294967295]
00000400:00000200:1.0:1788574638.486660:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574638.486665:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574638.486669:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574638.486671:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574638.486673:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574638.486675:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1788574638.486679:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574638.486681:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1788574638.486684:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574638.486685:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574638.486687:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574638.486690:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574638.486692:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574638.486693:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574638.486695:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574638.486697:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486703:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574638.486705:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486709:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574638.486710:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574638.486713:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574638.486717:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574638.486719:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574638.486720:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306242087
00000100:00000001:1.0:1788574638.486723:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306243001
00000100:00000001:1.0:1788574638.486724:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306242087
00000100:00000001:1.0:1788574638.486726:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306243001
00000100:00000001:1.0:1788574638.486728:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574638.486729:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574638.486731:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574638.486733:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574638/real 1788574638]  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574638.486738:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574638.486738:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574638.486739:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574638.486741:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574638.486742:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574638.486743:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574638.486745:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574638.486750:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486755:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574638.486756:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486757:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574638.486758:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574638.486760:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486764:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574638.486767:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486772:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574638.486774:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486775:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574638.486776:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486777:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574638.486779:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574638.486780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574638.486782:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574638.486784:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574638.486786:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574638.486791:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574638.486793:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574638.486795:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486799:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574638.486800:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574638.486802:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574638.486803:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574638.486804:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574638.486806:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574638.486808:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574638.486809:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574638.486810:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574638.486812:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574638.486814:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574638.486815:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574638.486818:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574638.486820:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574638.486823:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574638.486825:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393149696, portal 4
00000100:00000001:1.0:1788574638.486826:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574638.486827:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:1.0:1788574638.486832:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486836:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574638.486838:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574638.486839:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574638.486841:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393149696, offset 0
00000400:00000200:1.0:1788574638.486843:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574638.486845:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574638.486861:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574638.486864:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574638.486866:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574638.486867:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574638.486869:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574638.486871:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330425:330425:256:4294967295] 192.168.202.110@tcp LPNI seq info [330382:330382:8:4294967295]
00000400:00000200:1.0:1788574638.486874:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574638.486879:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574638.486882:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574638.486885:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574638.486888:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574638.486890:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1788574638.486893:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574638.486895:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1788574638.486897:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574638.486899:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574638.486901:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574638.486904:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574638.486906:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574638.486908:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574638.486910:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574638.486912:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486918:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574638.486920:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486925:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574638.486926:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574638.486927:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574638.486930:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574638.486932:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574638.486933:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306242233
00000100:00000001:1.0:1788574638.486934:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306243001
00000100:00000001:1.0:1788574638.486936:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306242233
00000100:00000001:1.0:1788574638.486938:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306243001
00000100:00000001:1.0:1788574638.486939:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574638.486941:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574638.486942:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574638.486944:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574638/real 1788574638]  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574638.486949:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574638.486950:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574638.486951:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:1.0:1788574638.486952:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:1.0:1788574638.486953:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:1.0:1788574638.486954:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574638.486956:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574638.486961:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486970:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574638.486971:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486973:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574638.486974:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574638.486976:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486980:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574638.486982:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.486986:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574638.486988:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486989:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574638.486989:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.486990:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574638.486992:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574638.486993:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574638.486995:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574638.486997:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574638.487000:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574638.487005:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574638.487007:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.487008:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574638.487010:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393149568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.487014:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574638.487015:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574638.487017:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574638.487018:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574638.487020:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574638.487021:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574638.487022:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574638.487023:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:1.0:1788574638.487025:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574638.487026:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574638.487027:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:1.0:1788574638.487029:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574638.487030:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.487031:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574638.487033:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393149696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574638.487038:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574638.487039:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574638.487039:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574638.487041:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574638.487041:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574638.487042:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574638.487043:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574638.487044:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:1.0:1788574638.487045:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574638.487046:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574638.487047:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:1.0:1788574638.487048:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574638.487049:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.487050:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574638.487052:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574638.487053:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574638.487054:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574638.487055:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574639.445509:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306243001, at 4306243008
00000100:00000010:0.0:1788574639.445524:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9b40.
00000100:00000040:1.0:1788574639.445546:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306243001, at 4306243008
00000100:00000010:1.0:1788574639.445562:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9240.
00000100:00000001:1.0:1788574643.605834:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574643.605840:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574643.605847:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574643.605859:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574643.605881:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574643.605886:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574643.605893:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574643.605898:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574643.605904:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574643.605913:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393149824:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574643.605921:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574643.605928:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574643.605933:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574643.605940:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574643.605948:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975ac00.
02000000:00000001:1.0:1788574643.605953:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574643.605960:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574643.605969:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d580.
00000400:00000010:1.0:1788574643.605980:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2dd0.
00000100:00000200:1.0:1788574643.605990:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393149824, portal 25
00000100:00000001:1.0:1788574643.605995:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574643.606000:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:1.0:1788574643.606010:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574643.606033:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574643.606037:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574643.606042:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc22a8.
00000100:00000200:1.0:1788574643.606047:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393149824, offset 0
00000400:00000200:1.0:1788574643.606056:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574643.606072:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574643.606083:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574643.606089:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574643.606096:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574643.606103:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574643.606109:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574643.606120:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330426:330426:256:4294967295] 192.168.202.110@tcp LPNI seq info [330383:330383:8:4294967295]
00000400:00000200:1.0:1788574643.606132:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574643.606149:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574643.606164:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574643.606172:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cea00.
00000800:00000200:1.0:1788574643.606179:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574643.606188:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574643.606204:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574643.606215:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574643.606221:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306248121
00000100:00000010:1.0:1788574643.606233:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9240.
00000100:00000001:1.0:1788574643.606244:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574643.606265:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306248121
00000100:00000010:1.0:1788574643.606273:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9f60.
00000100:00000001:1.0:1788574643.606279:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574643.606285:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574643.606290:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574643.606294:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574643.606297:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574643.606309:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574643.606313:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1788574643.606319:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574643.606322:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574643.606325:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574643.606334:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574643.606355:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574643.606358:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574643.606371:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574643.606375:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574643.606382:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574643.606386:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149952:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574643.606427:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574643.606430:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574643.606432:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574643.606434:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574643.606438:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759c00.
02000000:00000001:1.0:1788574643.606441:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574643.606445:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574643.606448:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d108.
00000400:00000010:1.0:1788574643.606453:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2b28.
00000100:00000200:1.0:1788574643.606456:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393149952, portal 10
00000100:00000001:1.0:1788574643.606458:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574643.606459:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530777728 : -130484178773888 : ffff895348e01880)
00000100:00000040:1.0:1788574643.606467:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574643.606475:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574643.606477:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574643.606480:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc23b8.
00000100:00000200:1.0:1788574643.606483:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393149952, offset 0
00000400:00000200:1.0:1788574643.606488:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574643.606493:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574643.606497:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574643.606500:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574643.606503:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574643.606506:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574643.606508:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574643.606512:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330427:330427:255:4294967295] 192.168.202.110@tcp LPNI seq info [330384:330384:7:4294967295]
00000400:00000200:1.0:1788574643.606516:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574643.606523:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574643.606528:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574643.606532:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656cee00.
00000800:00000200:1.0:1788574643.606536:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574643.606541:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574643.606544:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574643.606547:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574643.606551:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574643.606553:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306247207
00000100:00000001:1.0:1788574643.606556:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306248121
00000100:00000001:1.0:1788574643.606559:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306247207
00000100:00000001:1.0:1788574643.606561:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306248121
00000100:00000001:1.0:1788574643.606563:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574643.606566:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574643.606568:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574643.606570:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574643.606574:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574643.606575:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1788574643.606577:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574643.606579:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574643.606580:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574643.606585:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574643.606592:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574643.606594:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574643.606596:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574643.606598:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574643.606600:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574643.606603:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150080:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574643.606607:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574643.606609:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574643.606610:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574643.606612:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574643.606615:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379758000.
02000000:00000001:1.0:1788574643.606617:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574643.606621:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574643.606624:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5d7e8.
00000400:00000010:1.0:1788574643.606628:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc24c8.
00000100:00000200:1.0:1788574643.606631:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393150080, portal 4
00000100:00000001:1.0:1788574643.606633:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574643.606634:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:1.0:1788574643.606641:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574643.606648:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574643.606651:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574643.606654:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2cc0.
00000100:00000200:1.0:1788574643.606656:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393150080, offset 0
00000400:00000200:1.0:1788574643.606661:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574643.606665:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574643.606669:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574643.606672:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574643.606675:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574643.606677:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574643.606680:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574643.606683:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330428:330428:254:4294967295] 192.168.202.110@tcp LPNI seq info [330385:330385:6:4294967295]
00000400:00000200:1.0:1788574643.606688:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574643.606695:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574643.606700:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574643.606703:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574643.606707:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574643.606711:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574643.606714:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574643.606717:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574643.606720:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574643.606722:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306247353
00000100:00000001:1.0:1788574643.606725:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306248121
00000100:00000001:1.0:1788574643.606727:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306247353
00000100:00000001:1.0:1788574643.606729:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306248121
00000100:00000001:1.0:1788574643.606732:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574643.606734:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574643.606737:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574643.606739:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574643.606742:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574643.606744:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1788574643.606746:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574643.606747:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574643.606749:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574643.606751:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574644.565465:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306248121, at 4306248128
00000100:00000040:0.0:1788574644.565470:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306248121, at 4306248128
00000100:00000010:1.0:1788574644.565477:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9f60.
00000100:00000010:0.0:1788574644.565479:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9240.
00000400:00000100:1.0:1788574646.677904:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788574646.677972:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788574646.677984:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574646.677989:0:2351:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788574646.677994:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574646.677998:0:2351:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788574646.678004:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574646.678008:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cea00.
00000400:00000200:1.0:1788574646.678014:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574646.678018:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574646.678026:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc22a8
00000400:00000010:1.0:1788574646.678029:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc22a8.
00000100:00000001:1.0:1788574646.678035:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574646.678045:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574646.678088:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574646.678093:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574646.678101:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574646.678104:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574646.678118:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574646.678121:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656cee00.
00000400:00000200:1.0:1788574646.678124:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574646.678128:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574646.678131:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc23b8
00000400:00000010:1.0:1788574646.678133:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc23b8.
00000100:00000001:1.0:1788574646.678136:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574646.678140:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574646.678148:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574646.678152:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574646.678159:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574646.678161:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574646.678165:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574646.678168:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574646.678170:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574646.678174:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574646.678176:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2cc0
00000400:00000010:1.0:1788574646.678179:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2cc0.
00000100:00000001:1.0:1788574646.678181:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574646.678185:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574646.678193:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574646.678196:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574646.678202:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574646.678203:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574646.678295:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574646.678300:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574646.678303:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574646.678311:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574643/real 1788574646]  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574646.678324:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574646.678326:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574646.678330:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d7e8.
00000400:00000200:0.0:1788574646.678334:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc24c8
00000400:00000010:0.0:1788574646.678336:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc24c8.
00000100:00000001:0.0:1788574646.678339:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574646.678343:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574646.678353:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678361:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574646.678363:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678366:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574646.678368:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574646.678373:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678380:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574646.678386:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678517:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574646.678520:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574646.678525:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678528:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574646.678532:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574646.678536:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574646.678544:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574646.678549:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574646.678555:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574646.678564:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150080:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574646.678570:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574646.678573:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574643/real 1788574646]  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574646.678581:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574646.678582:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574646.678584:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d108.
00000400:00000200:0.0:1788574646.678594:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2b28
00000400:00000010:0.0:1788574646.678596:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2b28.
00000100:00000001:0.0:1788574646.678598:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574646.678602:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574646.678610:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678617:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574646.678619:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678620:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574646.678622:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574646.678627:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678641:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574646.678646:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678652:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574646.678655:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678657:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574646.678658:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678660:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574646.678663:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574646.678666:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574646.678670:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574646.678672:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574646.678677:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574646.678684:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393149952:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574646.678688:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574646.678692:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574643/real 1788574646]  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574646.678701:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574646.678702:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574646.678704:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5d580.
00000400:00000200:0.0:1788574646.678708:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2dd0
00000400:00000010:0.0:1788574646.678709:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2dd0.
00000100:00000001:0.0:1788574646.678712:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574646.678716:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574646.678731:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678737:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574646.678738:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678740:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574646.678741:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574646.678744:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678748:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574646.678751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678755:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574646.678756:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678757:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574646.678758:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678759:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574646.678760:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574646.678762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574646.678779:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574646.678782:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574646.678786:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574646.678794:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393149824:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574646.678798:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678800:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574646.678802:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393150080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678811:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574646.678813:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574646.678818:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379758000.
02000000:00000001:0.0:1788574646.678822:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574646.678825:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574646.678827:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574646.678830:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574646.678832:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788574646.678836:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574646.678838:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574646.678840:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:0.0:1788574646.678843:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574646.678845:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678847:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574646.678850:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01880 x1875444393149952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678858:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574646.678859:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574646.678861:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759c00.
02000000:00000001:0.0:1788574646.678864:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574646.678865:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574646.678866:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574646.678869:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574646.678870:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788574646.678873:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574646.678874:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574646.678875:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01880.
00000100:00000001:0.0:1788574646.678878:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574646.678878:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678880:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574646.678883:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393149824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574646.678887:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574646.678888:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574646.678890:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975ac00.
02000000:00000001:0.0:1788574646.678892:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574646.678892:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574646.678893:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574646.678895:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574646.678896:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574646.678897:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574646.678898:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574646.678900:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:0.0:1788574646.678902:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574646.678903:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678905:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574646.678913:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574646.678916:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574646.678919:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574646.678920:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574648.725548:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574648.725560:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574648.725563:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574648.725566:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574648.725571:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574648.725578:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11579
00000400:00000200:0.0:1788574648.725595:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574648.725602:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574648.725605:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574648.725608:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.725611:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574648.725614:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.725616:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574648.725619:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574648.725621:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574648.725624:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574648.725628:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574648.725630:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574648.725633:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574648.725636:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.725638:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574648.725640:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.725642:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574648.725645:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574648.725647:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574648.725649:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574648.725652:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574648.725655:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574648.725657:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574648.725659:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574648.725661:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574648.725671:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788574648.725674:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574648.725677:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574648.725679:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574648.725682:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574648.725685:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574648.725689:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574648.725697:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788574648.725706:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574648.725718:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574648.725731:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574648.725737:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574648.725758:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574648.725791:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574648.725794:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574648.725797:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574648.725799:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574648.725801:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574648.725803:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574648.725806:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11579
00000400:00000200:0.0:1788574648.725809:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574648.725812:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574648.725814:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574648.725816:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.725818:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574648.725820:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.725823:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574648.725826:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574648.725829:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574648.725831:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574648.725835:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574648.725837:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574648.725840:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574648.725845:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.725847:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574648.725851:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574648.725851:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574648.725854:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:1.0:1788574648.725858:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1788574648.725860:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:1.0:1788574648.725864:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1788574648.725866:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574648.725871:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574648.725875:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1788574648.725876:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788574648.725884:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574648.725891:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574648.725894:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574648.725896:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574648.725898:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1788574648.725904:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1788574648.725905:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:0.0:1788574648.725911:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574648.725915:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574648.725919:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788574648.725922:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574648.725925:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574648.725927:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574648.725927:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574648.725929:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574648.725934:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574648.725935:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574648.725939:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574648.725944:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393150208:192.168.202.110@tcp:250:kworker.0
02000000:00000010:0.0:1788574648.725948:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:1.0:1788574648.725953:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1788574648.725956:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574648.725964:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:0.0:1788574648.725965:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1788574648.725970:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574648.725978:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788574648.725982:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574648.725987:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574648.725989:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759400.
00000100:00000040:0.0:1788574648.725994:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788574648.725994:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574648.726009:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574648.726009:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788574648.726016:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1788574648.726019:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5ddc0.
00000100:00080000:0.0:1788574648.726022:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574648.726025:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574648.726028:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574648.726031:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:1.0:1788574648.726033:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2cc0.
00000100:00080000:0.0:1788574648.726035:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11579
00000400:00000200:0.0:1788574648.726043:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574648.726046:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000200:1.0:1788574648.726047:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393150208, portal 25
00000100:00080000:0.0:1788574648.726050:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574648.726055:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.726059:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1788574648.726061:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000040:0.0:1788574648.726063:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574648.726066:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00000001:0.0:1788574648.726069:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574648.726073:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574648.726078:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000040:1.0:1788574648.726081:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1788574648.726083:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574648.726086:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574648.726089:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574648.726092:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574648.726096:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.726102:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574648.726107:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574648.726111:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788574648.726112:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574648.726118:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000040:1.0:1788574648.726119:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788574648.726126:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:1.0:1788574648.726128:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc23b8.
00000100:00000001:0.0:1788574648.726132:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1788574648.726135:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393150208, offset 0
02000000:00000001:0.0:1788574648.726137:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574648.726142:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574648.726146:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574648.726148:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574648.726151:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:1.0:1788574648.726156:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00000001:0.0:1788574648.726157:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574648.726162:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574648.726168:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788574648.726170:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:1.0:1788574648.726174:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788574648.726176:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574648.726179:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574648.726182:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574648.726184:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1788574648.726188:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1788574648.726189:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574648.726197:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000400:00000200:1.0:1788574648.726203:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788574648.726204:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574648.726214:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574648.726223:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1788574648.726224:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1788574648.726226:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574648.726234:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1788574648.726235:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788574648.726243:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788574648.726253:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574648.726253:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330429:330429:256:4294967295] 192.168.202.110@tcp LPNI seq info [330386:330386:8:4294967295]
00000400:00000200:1.0:1788574648.726261:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788574648.726266:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574648.726273:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:1.0:1788574648.726282:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1788574648.726284:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11589)
00000800:00000200:1.0:1788574648.726292:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574648.726301:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574648.726310:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574648.726316:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1788574648.726324:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574648.726328:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1788574648.726334:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574648.726338:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574648.726345:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574648.726351:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574648.726359:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc23b8
00000400:00000010:1.0:1788574648.726363:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc23b8.
00000100:00000001:1.0:1788574648.726369:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574648.726374:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726386:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574648.726390:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574673 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726435:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574648.726437:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574648.726447:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574648.726452:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574648.726457:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306253242
00000100:00000010:1.0:1788574648.726464:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9f60.
00000100:00000001:1.0:1788574648.726475:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574648.726493:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306253242
00000100:00000010:1.0:1788574648.726498:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f94e0.
00000100:00000001:1.0:1788574648.726502:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574648.726506:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574648.726511:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574648.726519:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574648.726526:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574648/real 1788574648]  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574648.726538:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574648.726541:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574648.726544:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5ddc0.
00000400:00000200:1.0:1788574648.726547:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2cc0
00000400:00000010:1.0:1788574648.726549:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2cc0.
00000100:00000001:1.0:1788574648.726552:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574648.726556:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574648.726566:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726574:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574648.726576:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.726579:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574648.726582:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574648.726587:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726594:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574648.726600:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726608:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574648.726611:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.726613:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574648.726615:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.726618:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574648.726620:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574648.726624:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574648.726629:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574648.726633:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574648.726638:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574648.726646:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393150208:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574648.726653:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.726655:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574648.726659:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393150208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726665:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574648.726669:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574648.726672:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759400.
02000000:00000001:1.0:1788574648.726676:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574648.726678:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574648.726679:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574648.726682:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574648.726685:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:1.0:1788574648.726691:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574648.726693:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574648.726696:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:1.0:1788574648.726700:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574648.726701:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.726704:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.726714:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574648.726717:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574648.726719:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574648.726721:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574648.726722:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574648.726726:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726734:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574648.726735:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574648.726738:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574648.726740:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574648.726762:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574648.726765:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150336:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574648.726768:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574648.726771:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574648.726773:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574648.726775:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574648.726779:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759400.
02000000:00000001:1.0:1788574648.726781:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574648.726785:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574648.726788:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5ddc0.
00000400:00000010:1.0:1788574648.726792:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2cc0.
00000100:00000200:1.0:1788574648.726797:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393150336, portal 10
00000100:00000001:1.0:1788574648.726799:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574648.726800:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00000040:1.0:1788574648.726813:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726820:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574648.726822:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574648.726825:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc23b8.
00000100:00000200:1.0:1788574648.726827:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393150336, offset 0
00000400:00000200:1.0:1788574648.726831:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574648.726836:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574648.726841:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574648.726845:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574648.726848:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574648.726850:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574648.726853:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574648.726856:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330430:330430:256:4294967295] 192.168.202.110@tcp LPNI seq info [330387:330387:8:4294967295]
00000400:00000200:1.0:1788574648.726860:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574648.726867:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574648.726872:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574648.726877:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574648.726881:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574648.726885:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1788574648.726889:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574648.726892:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1788574648.726896:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574648.726898:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574648.726900:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574648.726903:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574648.726906:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc23b8
00000400:00000010:1.0:1788574648.726907:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc23b8.
00000100:00000001:1.0:1788574648.726909:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574648.726912:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726918:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574648.726920:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574673 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726925:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574648.726926:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574648.726928:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574648.726932:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574648.726934:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574648.726936:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306252327
00000100:00000001:1.0:1788574648.726939:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306253242
00000100:00000001:1.0:1788574648.726940:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306252327
00000100:00000001:1.0:1788574648.726942:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306253242
00000100:00000001:1.0:1788574648.726944:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574648.726945:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574648.726947:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574648.726949:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574648/real 1788574648]  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574648.726962:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574648.726963:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574648.726966:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5ddc0.
00000400:00000200:1.0:1788574648.726968:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2cc0
00000400:00000010:1.0:1788574648.726970:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2cc0.
00000100:00000001:1.0:1788574648.726972:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574648.726976:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574648.726985:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.726993:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574648.726995:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.726997:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574648.726998:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574648.727003:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727011:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574648.727016:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727023:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574648.727025:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.727027:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574648.727028:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.727029:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574648.727031:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574648.727033:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574648.727036:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574648.727038:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574648.727044:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574648.727052:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150336:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574648.727056:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574648.727060:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727067:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574648.727069:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574648.727072:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574648.727074:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574648.727076:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574648.727079:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574648.727083:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574648.727085:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574648.727087:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574648.727089:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574648.727093:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975a800.
02000000:00000001:1.0:1788574648.727096:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574648.727101:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574648.727104:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5ddc0.
00000400:00000010:1.0:1788574648.727108:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2cc0.
00000100:00000200:1.0:1788574648.727110:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393150464, portal 4
00000100:00000001:1.0:1788574648.727112:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574648.727113:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523505920 : -130484186045696 : ffff895348712300)
00000100:00000040:1.0:1788574648.727118:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727123:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574648.727125:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574648.727126:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc23b8.
00000100:00000200:1.0:1788574648.727128:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393150464, offset 0
00000400:00000200:1.0:1788574648.727131:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574648.727134:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574648.727137:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574648.727140:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574648.727142:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574648.727143:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574648.727145:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574648.727148:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330431:330431:256:4294967295] 192.168.202.110@tcp LPNI seq info [330388:330388:8:4294967295]
00000400:00000200:1.0:1788574648.727151:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574648.727156:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574648.727159:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574648.727162:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574648.727165:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574648.727167:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1788574648.727170:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574648.727172:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1788574648.727175:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574648.727176:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574648.727179:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574648.727181:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574648.727183:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc23b8
00000400:00000010:1.0:1788574648.727184:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc23b8.
00000100:00000001:1.0:1788574648.727186:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574648.727188:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727194:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574648.727196:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574673 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727200:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574648.727201:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574648.727203:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574648.727205:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574648.727207:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574648.727209:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306252473
00000100:00000001:1.0:1788574648.727211:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306253242
00000100:00000001:1.0:1788574648.727213:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306252473
00000100:00000001:1.0:1788574648.727214:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306253242
00000100:00000001:1.0:1788574648.727216:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574648.727217:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574648.727218:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574648.727220:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574648/real 1788574648]  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574648.727225:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574648.727227:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574648.727228:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5ddc0.
00000400:00000200:1.0:1788574648.727229:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2cc0
00000400:00000010:1.0:1788574648.727231:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2cc0.
00000100:00000001:1.0:1788574648.727232:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574648.727234:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574648.727240:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727245:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574648.727247:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.727248:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574648.727250:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574648.727254:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727262:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574648.727266:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727273:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574648.727276:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.727278:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574648.727279:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.727281:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574648.727283:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574648.727286:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574648.727290:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574648.727292:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574648.727296:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574648.727306:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574648.727309:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.727311:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574648.727313:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393150336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727318:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574648.727320:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574648.727321:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759400.
02000000:00000001:1.0:1788574648.727323:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574648.727324:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574648.727325:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574648.727327:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574648.727328:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:1.0:1788574648.727331:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574648.727332:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574648.727333:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:1.0:1788574648.727335:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574648.727336:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.727338:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574648.727341:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712300 x1875444393150464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574673 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574648.727354:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574648.727355:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574648.727357:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975a800.
02000000:00000001:1.0:1788574648.727360:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574648.727361:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574648.727362:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574648.727365:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574648.727366:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:1.0:1788574648.727369:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574648.727370:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574648.727372:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712300.
00000100:00000001:1.0:1788574648.727375:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574648.727376:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.727378:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574648.727383:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574648.727384:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574648.727386:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574648.727388:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574649.685473:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306253242, at 4306253248
00000100:00000040:1.0:1788574649.685480:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306253242, at 4306253248
00000100:00000010:0.0:1788574649.685493:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9f60.
00000100:00000010:1.0:1788574649.685497:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f94e0.
00000100:00080000:0.0:1788574653.845572:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574653.845586:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574653.845590:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574653.845596:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574653.845602:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574653.845612:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11584
00000400:00000200:0.0:1788574653.845624:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574653.845633:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574653.845636:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574653.845640:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.845644:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574653.845647:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.845651:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574653.845656:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574653.845659:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574653.845662:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574653.845666:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574653.845668:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574653.845671:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574653.845674:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.845676:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574653.845679:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.845681:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574653.845686:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574653.845690:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574653.845705:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574653.845709:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574653.845713:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574653.845717:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574653.845720:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574653.845723:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574653.845736:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788574653.845740:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574653.845745:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574653.845747:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574653.845751:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574653.845755:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574653.845766:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574653.845776:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788574653.845785:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574653.845796:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574653.845809:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574653.845815:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574653.845820:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574653.845844:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574653.845847:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574653.845851:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574653.845853:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574653.845855:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574653.845858:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574653.845862:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11584
00000400:00000200:0.0:1788574653.845866:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574653.845869:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574653.845872:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574653.845877:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.845879:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574653.845882:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.845885:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574653.845888:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574653.845890:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574653.845892:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574653.845895:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574653.845897:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574653.845900:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574653.845903:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.845906:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574653.845908:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.845910:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574653.845915:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:1.0:1788574653.845915:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1788574653.845925:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574653.845926:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574653.845932:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.845934:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:0.0:1788574653.845937:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574653.845945:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574653.845952:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574653.845954:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574653.845956:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574653.845965:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574653.845976:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.845980:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574653.845981:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.845987:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574653.845989:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788574653.845995:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574653.845996:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574653.846005:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574653.846007:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574653.846009:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574653.846016:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574653.846016:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574653.846020:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574653.846025:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574653.846030:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393150592:192.168.202.110@tcp:250:kworker.0
02000000:00000010:0.0:1788574653.846033:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574653.846042:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.846043:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574653.846053:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:0.0:1788574653.846056:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1788574653.846060:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574653.846069:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788574653.846078:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574653.846081:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574653.846085:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975a800.
00000100:00000040:0.0:1788574653.846088:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788574653.846090:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574653.846107:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574653.846107:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788574653.846116:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1788574653.846120:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5ddc0.
00000100:00080000:0.0:1788574653.846122:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574653.846125:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574653.846128:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574653.846132:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:1.0:1788574653.846134:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2cc0.
00000100:00080000:0.0:1788574653.846137:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11584
00000400:00000200:0.0:1788574653.846148:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:1.0:1788574653.846151:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393150592, portal 25
00000400:00000200:0.0:1788574653.846153:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1788574653.846159:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1788574653.846160:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1788574653.846163:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523500544 : -130484186051072 : ffff895348710e00)
00000100:00000040:0.0:1788574653.846167:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.846174:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574653.846182:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:1.0:1788574653.846184:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574653.846189:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574653.846196:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574653.846200:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574653.846205:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574653.846211:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:1.0:1788574653.846215:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:0.0:1788574653.846217:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574653.846222:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574653.846226:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574653.846233:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc23b8.
00000100:00000040:0.0:1788574653.846234:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.846242:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1788574653.846245:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393150592, offset 0
00000100:00000040:0.0:1788574653.846246:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574653.846255:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574653.846259:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:1.0:1788574653.846267:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788574653.846276:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1788574653.846283:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574653.846285:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574653.846291:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574653.846294:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574653.846298:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574653.846302:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788574653.846303:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574653.846308:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574653.846315:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574653.846315:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788574653.846322:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574653.846329:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000010:0.0:1788574653.846333:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000400:00000200:1.0:1788574653.846337:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000040:0.0:1788574653.846342:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:1.0:1788574653.846346:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:0.0:1788574653.846349:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574653.846352:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574653.846359:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1788574653.846359:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330432:330432:256:4294967295] 192.168.202.110@tcp LPNI seq info [330389:330389:8:4294967295]
02000000:00000001:0.0:1788574653.846363:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574653.846368:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574653.846377:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1788574653.846380:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574653.846388:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574653.846431:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1788574653.846451:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788574653.846455:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574653.846460:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574653.846465:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788574653.846471:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:1.0:1788574653.846476:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000100:00000001:0.0:1788574653.846490:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574653.846490:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574653.846500:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000040:0.0:1788574653.846501:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574653.846513:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:1.1:1788574653.846523:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574653.846528:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000040:0.0:1788574653.846529:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11595)
00000800:00000100:1.0:1788574653.846547:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574653.846552:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574653.846559:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574653.846567:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574653.846573:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc23b8
00000400:00000010:1.0:1788574653.846576:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc23b8.
00000100:00000001:1.0:1788574653.846581:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574653.846586:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.846598:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574653.846602:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574678 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.846610:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.846612:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574653.846618:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574653.846623:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574653.846626:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306258362
00000100:00000010:1.0:1788574653.846632:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f94e0.
00000100:00000001:1.0:1788574653.846639:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574653.846646:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306258362
00000100:00000010:1.0:1788574653.846649:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c8f9d80.
00000100:00000001:1.0:1788574653.846652:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574653.846655:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574653.846658:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.846660:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574653.846665:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574653/real 1788574653]  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574653.846675:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574653.846677:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574653.846679:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5ddc0.
00000400:00000200:1.0:1788574653.846682:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2cc0
00000400:00000010:1.0:1788574653.846684:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2cc0.
00000100:00000001:1.0:1788574653.846686:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574653.846703:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574653.846713:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.846720:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574653.846722:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.846724:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574653.846726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574653.846731:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.846738:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574653.846743:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.846750:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574653.846753:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.846754:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574653.846756:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.846758:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574653.846761:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574653.846765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574653.846769:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574653.846772:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574653.846776:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574653.846785:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393150592:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788574653.846790:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.846792:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574653.846795:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710e00 x1875444393150592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.846808:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574653.846811:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574653.846813:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975a800.
02000000:00000001:1.0:1788574653.846817:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574653.846819:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574653.846821:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788574653.846823:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574653.846826:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:1.0:1788574653.846830:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574653.846832:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574653.846834:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710e00.
00000100:00000001:1.0:1788574653.846838:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574653.846839:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.846842:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.846849:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574653.846851:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.846853:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574653.846855:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574653.846857:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574653.846861:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.846868:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574653.846870:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574653.846872:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574653.846874:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574653.846876:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574653.846880:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574653.846883:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574653.846885:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574653.846887:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574653.846889:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574653.846892:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff89537975a800.
02000000:00000001:1.0:1788574653.846894:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574653.846898:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574653.846901:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5ddc0.
00000400:00000010:1.0:1788574653.846905:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2cc0.
00000100:00000200:1.0:1788574653.846908:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393150720, portal 10
00000100:00000001:1.0:1788574653.846910:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574653.846911:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:1.0:1788574653.846919:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.846925:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574653.846927:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574653.846929:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc23b8.
00000100:00000200:1.0:1788574653.846932:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393150720, offset 0
00000400:00000200:1.0:1788574653.846936:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574653.846941:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574653.846944:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574653.846947:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574653.846950:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574653.846953:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574653.846956:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574653.846959:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330433:330433:256:4294967295] 192.168.202.110@tcp LPNI seq info [330390:330390:8:4294967295]
00000400:00000200:1.0:1788574653.846963:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574653.846969:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574653.846973:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574653.846976:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574653.846981:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574653.846983:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1788574653.846988:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574653.846991:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:1.0:1788574653.846995:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574653.846997:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574653.847000:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574653.847004:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574653.847007:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc23b8
00000400:00000010:1.0:1788574653.847008:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc23b8.
00000100:00000001:1.0:1788574653.847011:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574653.847015:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847022:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574653.847025:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574678 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847032:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.847033:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574653.847036:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574653.847039:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:1.0:1788574653.847043:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574653.847046:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306257447
00000100:00000001:1.0:1788574653.847048:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306258362
00000100:00000001:1.0:1788574653.847050:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306257447
00000100:00000001:1.0:1788574653.847052:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306258362
00000100:00000001:1.0:1788574653.847055:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574653.847057:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.847059:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574653.847062:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574653/real 1788574653]  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574653.847070:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574653.847072:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574653.847073:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5ddc0.
00000400:00000200:1.0:1788574653.847075:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2cc0
00000400:00000010:1.0:1788574653.847077:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2cc0.
00000100:00000001:1.0:1788574653.847079:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574653.847082:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574653.847090:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847102:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574653.847104:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847106:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574653.847107:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574653.847111:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847118:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574653.847122:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847129:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574653.847132:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847134:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574653.847135:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847137:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574653.847139:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574653.847142:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574653.847145:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574653.847148:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574653.847152:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574653.847160:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788574653.847164:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788574653.847167:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847174:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574653.847176:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574653.847178:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788574653.847180:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574653.847182:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574653.847185:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150848:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574653.847188:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574653.847191:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788574653.847192:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574653.847194:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574653.847197:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895379759400.
02000000:00000001:1.0:1788574653.847199:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788574653.847203:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1788574653.847205:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff895342f5ddc0.
00000400:00000010:1.0:1788574653.847208:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc2cc0.
00000100:00000200:1.0:1788574653.847211:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393150848, portal 4
00000100:00000001:1.0:1788574653.847213:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788574653.847215:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523505920 : -130484186045696 : ffff895348712300)
00000100:00000040:1.0:1788574653.847220:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847227:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788574653.847229:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1788574653.847231:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895372cc23b8.
00000100:00000200:1.0:1788574653.847233:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393150848, offset 0
00000400:00000200:1.0:1788574653.847237:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788574653.847241:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574653.847244:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788574653.847247:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788574653.847250:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788574653.847252:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788574653.847255:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1788574653.847258:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330434:330434:256:4294967295] 192.168.202.110@tcp LPNI seq info [330391:330391:8:4294967295]
00000400:00000200:1.0:1788574653.847262:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574653.847268:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788574653.847273:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574653.847276:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8953656ce900.
00000800:00000200:1.0:1788574653.847280:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1788574653.847282:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1788574653.847287:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1788574653.847289:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:1.0:1788574653.847293:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1788574653.847295:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8953656ce900.
00000400:00000200:1.0:1788574653.847298:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574653.847302:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788574653.847305:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc23b8
00000400:00000010:1.0:1788574653.847306:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc23b8.
00000100:00000001:1.0:1788574653.847309:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574653.847313:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847320:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574653.847323:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574678 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847329:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.847331:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574653.847334:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574653.847337:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:1.0:1788574653.847341:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574653.847344:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306257593
00000100:00000001:1.0:1788574653.847346:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306258362
00000100:00000001:1.0:1788574653.847348:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306257593
00000100:00000001:1.0:1788574653.847350:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306258362
00000100:00000001:1.0:1788574653.847352:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574653.847354:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.847356:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1788574653.847359:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574653/real 1788574653]  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1788574653.847367:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1788574653.847369:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1788574653.847370:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff895342f5ddc0.
00000400:00000200:1.0:1788574653.847373:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895372cc2cc0
00000400:00000010:1.0:1788574653.847375:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895372cc2cc0.
00000100:00000001:1.0:1788574653.847377:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:1.0:1788574653.847380:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788574653.847388:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847418:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:1.0:1788574653.847420:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847422:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574653.847423:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574653.847427:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847434:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574653.847438:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847446:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788574653.847448:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847450:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574653.847451:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847453:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1788574653.847454:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1788574653.847457:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1788574653.847460:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574653.847463:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574653.847468:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574653.847476:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393150848:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1788574653.847480:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847482:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574653.847485:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393150720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847492:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574653.847493:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574653.847495:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff89537975a800.
02000000:00000001:1.0:1788574653.847498:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574653.847500:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574653.847501:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:1.0:1788574653.847503:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574653.847505:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:1.0:1788574653.847508:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574653.847509:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574653.847511:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:1.0:1788574653.847514:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574653.847515:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847516:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574653.847520:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712300 x1875444393150848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574678 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574653.847526:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788574653.847527:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788574653.847528:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895379759400.
02000000:00000001:1.0:1788574653.847530:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788574653.847531:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788574653.847533:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:1.0:1788574653.847535:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788574653.847536:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:1.0:1788574653.847538:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788574653.847540:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788574653.847541:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712300.
00000100:00000001:1.0:1788574653.847544:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788574653.847545:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847547:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574653.847550:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574653.847552:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574653.847554:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788574653.847556:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574654.805475:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306258362, at 4306258368
00000100:00000040:1.0:1788574654.805475:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306258362, at 4306258368
00000100:00000010:0.0:1788574654.805490:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f94e0.
00000100:00000010:1.0:1788574654.805494:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c8f9d80.
00000100:00080000:1.0:1788574658.965551:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574658.965566:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574658.965570:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574658.965574:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574658.965581:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574658.965590:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11590
00000400:00000200:1.0:1788574658.965601:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574658.965610:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574658.965614:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574658.965618:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574658.965622:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574658.965625:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574658.965628:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574658.965633:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574658.965635:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574658.965686:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574658.965691:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574658.965695:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574658.965699:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574658.965703:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574658.965706:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574658.965709:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574658.965711:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574658.965715:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574658.965720:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574658.965724:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574658.965727:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574658.965732:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574658.965736:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574658.965739:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574658.965742:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574658.965758:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788574658.965763:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574658.965769:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574658.965771:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574658.965775:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574658.965778:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574658.965785:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574658.965796:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788574658.965805:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574658.965834:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574658.965847:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574658.965853:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574658.965859:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574658.965901:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574658.965905:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574658.965910:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574658.965912:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574658.965914:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574658.965918:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574658.965923:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11590
00000100:00000001:0.0:1788574658.965925:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574658.965931:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000400:00000200:1.0:1788574658.965931:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574658.965936:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000400:00000200:1.0:1788574658.965937:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574658.965942:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574658.965946:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788574658.965949:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574658.965956:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574658.965961:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574658.965962:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574658.965965:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574658.965966:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:0.0:1788574658.965969:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:1.0:1788574658.965972:0:188751:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:0.0:1788574658.965973:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1788574658.965976:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
02000000:00000001:0.0:1788574658.965977:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574658.965981:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:0.0:1788574658.965984:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393150976:192.168.202.110@tcp:250:kworker.0
00000020:00000040:1.0:1788574658.965986:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:0.0:1788574658.965990:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:1.0:1788574658.965992:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:0.0:1788574658.965995:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:1.0:1788574658.965996:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:0.0:1788574658.965998:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574658.966003:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:1.0:1788574658.966006:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574658.966009:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000010:0.0:1788574658.966010:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
00000100:00000040:1.0:1788574658.966014:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:0.0:1788574658.966015:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574658.966021:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788574658.966026:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:1.0:1788574658.966028:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000400:00000010:0.0:1788574658.966031:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000100:00080000:1.0:1788574658.966033:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574658.966038:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574658.966040:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
02000000:00000001:1.0:1788574658.966041:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:0.0:1788574658.966050:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393150976, portal 25
02000000:00000001:1.0:1788574658.966051:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574658.966054:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000001:1.0:1788574658.966056:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966057:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
00000100:00000001:1.0:1788574658.966062:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:0.0:1788574658.966066:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:1.0:1788574658.966070:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1788574658.966075:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00000001:1.0:1788574658.966076:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574658.966082:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574658.966082:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574658.966086:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574658.966090:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393150976, offset 0
00000100:00000010:1.0:1788574658.966090:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788574658.966097:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:0.0:1788574658.966103:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788574658.966104:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574658.966107:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574658.966110:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574658.966112:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1788574658.966114:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788574658.966116:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574658.966123:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000010:1.0:1788574658.966123:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000400:00000200:0.0:1788574658.966131:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1788574658.966131:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574658.966140:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00100000:1.0:1788574658.966142:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788574658.966146:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574658.966152:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574658.966159:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330435:330435:256:4294967295] 192.168.202.110@tcp LPNI seq info [330392:330392:8:4294967295]
00000100:00000001:1.0:1788574658.966163:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1788574658.966165:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574658.966169:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574658.966179:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788574658.966180:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574658.966209:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574658.966218:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000100:00000001:1.0:1788574658.966220:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574658.966225:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.0:1788574658.966228:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:1.0:1788574658.966236:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:0.0:1788574658.966237:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:1.0:1788574658.966242:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1788574658.966246:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:1.0:1788574658.966248:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.1:1788574658.966252:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:1.0:1788574658.966255:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000100:0.0:1788574658.966275:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574658.966279:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000100:00080000:1.0:1788574658.966279:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11590
00000400:00000200:1.0:1788574658.966287:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574658.966288:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788574658.966293:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1788574658.966296:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:1.0:1788574658.966298:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1788574658.966302:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574658.966306:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000040:1.0:1788574658.966306:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574658.966311:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788574658.966312:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:1.0:1788574658.966318:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:0.0:1788574658.966319:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574658.966324:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574658.966330:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1788574658.966334:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000020:00000040:1.0:1788574658.966334:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574658.966339:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:0.0:1788574658.966340:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574683 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788574658.966344:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:0.0:1788574658.966349:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788574658.966351:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:0.0:1788574658.966352:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000001:1.0:1788574658.966357:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:0.0:1788574658.966360:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574658.966363:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574658.966368:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574658.966369:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788574658.966372:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306263481
00000100:00000040:1.0:1788574658.966374:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574658.966379:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000010:0.0:1788574658.966381:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000020:00000040:1.0:1788574658.966382:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574658.966388:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574658.966391:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788574658.966392:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574658.966397:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788574658.966400:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306263481
02000000:00000001:1.0:1788574658.966401:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:0.0:1788574658.966405:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7360.
02000000:00000001:1.0:1788574658.966407:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966410:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788574658.966412:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574658.966414:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00100000:1.0:1788574658.966416:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:0.0:1788574658.966417:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966458:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574658.966464:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574658/real 1788574658]  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1788574658.966465:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574658.966472:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574658.966482:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000400:00000001:0.0:1788574658.966483:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574658.966487:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:1.0:1788574658.966489:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000010:0.0:1788574658.966490:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574658.966495:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
02000000:00000001:1.0:1788574658.966497:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:0.0:1788574658.966499:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
02000000:00000001:1.0:1788574658.966501:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966503:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574658.966508:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574658.966508:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574658.966513:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574658.966518:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574658.966521:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788574658.966527:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:0.0:1788574658.966528:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574658.966531:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.966535:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1788574658.966536:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966537:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574658.966545:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788574658.966550:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574658.966555:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574658.966564:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574658.966567:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574658.966573:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966577:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574658.966581:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.966583:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:1.0:1788574658.966584:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.966585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.966589:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574658.966591:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574658.966598:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574658.966622:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1788574658.966624:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966625:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788574658.966633:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574658.966634:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788574658.966661:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574658.966668:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11600)
00000100:00100000:0.0:1788574658.966766:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393150976:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574658.966771:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.966773:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574658.966776:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393150976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.966781:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574658.966783:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574658.966786:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574658.966788:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574658.966790:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574658.966791:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574658.966793:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574658.966796:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:0.0:1788574658.966799:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574658.966801:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574658.966802:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:0.0:1788574658.966804:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574658.966805:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.966807:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.966815:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574658.966816:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966818:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574658.966819:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574658.966820:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574658.966823:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.966828:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574658.966829:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574658.966831:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574658.966832:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574658.966834:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574658.966836:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151104:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574658.966838:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574658.966840:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574658.966841:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574658.966842:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574658.966845:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574658.966846:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574658.966849:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574658.966851:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574658.966854:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574658.966857:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393151104, portal 10
00000100:00000001:0.0:1788574658.966859:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574658.966860:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000040:0.0:1788574658.966865:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.966877:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574658.966879:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574658.966881:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574658.966885:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393151104, offset 0
00000400:00000200:0.0:1788574658.966888:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574658.966892:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574658.966895:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574658.966898:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574658.966900:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574658.966902:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574658.966903:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574658.966906:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330436:330436:256:4294967295] 192.168.202.110@tcp LPNI seq info [330393:330393:8:4294967295]
00000400:00000200:0.0:1788574658.966908:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574658.966913:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574658.966916:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574658.966919:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574658.966921:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574658.966923:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1788574658.966927:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574658.966928:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1788574658.966932:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574658.966933:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574658.966936:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574658.966938:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574658.966940:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574658.966942:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574658.966944:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574658.966947:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.966952:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574658.966954:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574683 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.966959:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966960:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574658.966962:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574658.966965:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574658.966967:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574658.966968:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306262567
00000100:00000001:0.0:1788574658.966971:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306263481
00000100:00000001:0.0:1788574658.966973:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306262567
00000100:00000001:0.0:1788574658.966974:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306263481
00000100:00000001:0.0:1788574658.966976:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574658.966977:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.966978:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574658.966981:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574658/real 1788574658]  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574658.966985:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574658.966986:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574658.966987:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574658.966989:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574658.966990:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574658.966991:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574658.966993:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574658.966999:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967003:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574658.967004:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967005:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574658.967007:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574658.967009:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967013:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574658.967016:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967020:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574658.967022:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967023:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574658.967024:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967026:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574658.967028:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574658.967030:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574658.967032:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574658.967033:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574658.967036:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574658.967041:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151104:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574658.967044:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574658.967046:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967050:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574658.967051:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574658.967052:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574658.967053:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574658.967055:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574658.967056:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151232:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574658.967058:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574658.967059:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574658.967060:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574658.967061:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574658.967064:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574658.967065:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574658.967068:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574658.967069:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574658.967072:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574658.967074:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393151232, portal 4
00000100:00000001:0.0:1788574658.967075:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574658.967077:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:0.0:1788574658.967081:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967085:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574658.967087:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574658.967088:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574658.967090:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393151232, offset 0
00000400:00000200:0.0:1788574658.967092:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574658.967094:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574658.967096:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574658.967098:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574658.967100:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574658.967102:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574658.967103:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574658.967105:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330437:330437:256:4294967295] 192.168.202.110@tcp LPNI seq info [330394:330394:8:4294967295]
00000400:00000200:0.0:1788574658.967108:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574658.967112:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574658.967115:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574658.967116:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574658.967119:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574658.967121:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1788574658.967123:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574658.967125:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1788574658.967128:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574658.967129:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574658.967131:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574658.967133:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574658.967135:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574658.967136:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574658.967138:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574658.967140:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967145:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574658.967147:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574683 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967152:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.967153:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574658.967155:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574658.967157:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574658.967159:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574658.967160:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306262713
00000100:00000001:0.0:1788574658.967162:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306263481
00000100:00000001:0.0:1788574658.967163:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306262713
00000100:00000001:0.0:1788574658.967165:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306263481
00000100:00000001:0.0:1788574658.967166:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574658.967168:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.967169:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574658.967171:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574658/real 1788574658]  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574658.967175:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574658.967176:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574658.967177:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574658.967178:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574658.967179:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574658.967180:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574658.967182:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574658.967187:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967192:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574658.967193:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967194:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574658.967195:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574658.967197:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967202:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574658.967205:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967209:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574658.967210:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967211:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574658.967212:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967213:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574658.967215:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574658.967216:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574658.967218:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574658.967219:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574658.967221:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574658.967226:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151232:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574658.967228:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967229:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574658.967231:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393151104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967235:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574658.967236:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574658.967237:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574658.967239:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574658.967240:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574658.967241:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574658.967242:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574658.967244:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:0.0:1788574658.967246:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574658.967247:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574658.967248:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:0.0:1788574658.967250:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574658.967251:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967252:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574658.967254:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393151232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574683 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574658.967259:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574658.967260:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574658.967260:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574658.967262:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574658.967263:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574658.967263:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574658.967264:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574658.967265:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:0.0:1788574658.967266:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574658.967267:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574658.967268:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:0.0:1788574658.967269:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574658.967270:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967271:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574658.967273:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574658.967274:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574658.967275:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574658.967276:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574659.925533:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306263481, at 4306263488
00000100:00000040:1.0:1788574659.925533:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306263481, at 4306263488
00000100:00000010:1.0:1788574659.925552:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7360.
00000100:00000010:0.0:1788574659.925553:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00080000:1.0:1788574664.085472:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574664.085481:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574664.085483:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574664.085487:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574664.085491:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574664.085497:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11595
00000400:00000200:1.0:1788574664.085504:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574664.085510:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574664.085512:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574664.085514:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574664.085516:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574664.085518:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574664.085520:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574664.085523:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574664.085525:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574664.085527:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574664.085529:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574664.085531:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574664.085532:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574664.085534:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574664.085536:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574664.085537:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574664.085539:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574664.085541:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574664.085545:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574664.085547:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574664.085549:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574664.085552:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574664.085554:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574664.085556:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574664.085558:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574664.085566:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788574664.085569:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574664.085572:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574664.085573:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574664.085576:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574664.085578:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574664.085582:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574664.085615:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574664.085621:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574664.085629:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574664.085637:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574664.085640:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574664.085644:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574664.085660:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574664.085663:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574664.085666:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574664.085667:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574664.085669:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574664.085673:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:0.0:1788574664.085681:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1788574664.085683:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11595
00000100:00000001:0.0:1788574664.085688:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574664.085693:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000400:00000200:1.0:1788574664.085693:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574664.085701:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:0.0:1788574664.085705:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788574664.085705:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574664.085714:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574664.085720:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574664.085725:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574664.085727:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574664.085731:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574664.085731:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574664.085737:0:188751:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:0.0:1788574664.085739:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574664.085742:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574664.085744:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788574664.085748:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00100000:0.0:1788574664.085750:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393151360:192.168.202.110@tcp:250:kworker.0
00000020:00000001:1.0:1788574664.085752:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788574664.085757:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:1.0:1788574664.085758:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
02000000:00000001:0.0:1788574664.085761:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:1.0:1788574664.085763:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:0.0:1788574664.085764:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574664.085769:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:1.0:1788574664.085770:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000010:0.0:1788574664.085775:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
00000100:00000040:1.0:1788574664.085777:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:0.0:1788574664.085778:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574664.085784:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788574664.085787:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788574664.085789:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:0.0:1788574664.085792:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000100:00000001:1.0:1788574664.085794:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:0.0:1788574664.085798:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000020:00000040:1.0:1788574664.085800:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000200:0.0:1788574664.085805:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393151360, portal 25
00000100:00080000:1.0:1788574664.085806:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574664.085809:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574664.085813:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00000001:1.0:1788574664.085813:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574664.085818:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1788574664.085819:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.085826:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:1.0:1788574664.085827:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1788574664.085829:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574664.085833:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
02000000:00000001:1.0:1788574664.085834:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574664.085836:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393151360, offset 0
00000100:00000001:1.0:1788574664.085839:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1788574664.085843:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00100000:1.0:1788574664.085846:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1788574664.085851:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:1.0:1788574664.085852:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574664.085856:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574664.085859:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000010:1.0:1788574664.085863:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000400:00000200:0.0:1788574664.085864:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:1.0:1788574664.085870:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:0.0:1788574664.085871:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574664.085876:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1788574664.085877:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1788574664.085880:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:1.0:1788574664.085881:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574664.085886:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330438:330438:256:4294967295] 192.168.202.110@tcp LPNI seq info [330395:330395:8:4294967295]
00000100:00000001:1.0:1788574664.085887:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574664.085891:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1788574664.085892:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788574664.085896:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574664.085903:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000010:1.0:1788574664.085904:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000800:00000200:0.0:1788574664.085910:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574664.085911:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788574664.085915:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000100:00100000:1.0:1788574664.085922:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1788574664.085923:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574664.085929:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788574664.085940:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574664.085942:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:0.1:1788574664.085943:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:1.0:1788574664.085948:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1788574664.085951:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574664.085955:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000100:00000040:1.0:1788574664.085959:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788574664.085961:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574664.085971:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1788574664.085973:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574664.085976:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000100:00080000:1.0:1788574664.085977:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1788574664.085980:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00080000:1.0:1788574664.085982:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574664.085985:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574664.085989:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574664.085993:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574664.085996:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574664.085998:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1788574664.086000:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:0.0:1788574664.086003:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574689 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788574664.086006:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11595
00000400:00000200:1.0:1788574664.086015:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574664.086016:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086019:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000400:00000200:1.0:1788574664.086020:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574664.086022:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1788574664.086024:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574664.086026:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574664.086030:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574664.086030:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574664.086034:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574664.086035:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306268601
00000100:00000001:1.0:1788574664.086036:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574664.086039:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000010:0.0:1788574664.086041:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000020:00000040:1.0:1788574664.086041:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574664.086042:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574664.086047:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574664.086048:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:0.0:1788574664.086049:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:1.0:1788574664.086050:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574664.086055:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574664.086056:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306268601
00000100:00000001:1.0:1788574664.086057:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000010:0.0:1788574664.086063:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d79c0.
00000100:00000040:1.0:1788574664.086065:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574664.086067:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574664.086072:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788574664.086072:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788574664.086078:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788574664.086079:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574664.086084:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574664.086087:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086090:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
02000000:00000001:1.0:1788574664.086091:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1788574664.086096:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574664/real 1788574664]  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788574664.086097:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574664.086103:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574664.086109:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574664.086113:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574664.086117:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1788574664.086118:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1788574664.086119:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1788574664.086121:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000010:1.0:1788574664.086124:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000400:00000010:0.0:1788574664.086128:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000020:00000040:1.0:1788574664.086129:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:0.0:1788574664.086133:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
02000000:00000001:1.0:1788574664.086134:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:0.0:1788574664.086137:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
02000000:00000001:1.0:1788574664.086138:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086142:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000001:1.0:1788574664.086142:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574664.086144:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574664.086147:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574664.086148:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788574664.086151:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574664.086157:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788574664.086160:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086167:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574664.086168:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086170:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574664.086171:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788574664.086173:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1788574664.086177:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574664.086193:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574664.086196:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086198:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574664.086200:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788574664.086206:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574664.086214:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086220:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:1.0:1788574664.086220:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1788574664.086227:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574664.086227:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:0.0:1788574664.086231:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574664.086233:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788574664.086235:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11605)
00000100:00000001:0.0:1788574664.086236:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574664.086240:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574664.086242:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574664.086249:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574664.086251:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574664.086257:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574664.086264:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393151360:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574664.086267:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086269:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574664.086271:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393151360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086274:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574664.086277:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574664.086279:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574664.086281:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574664.086282:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574664.086283:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574664.086285:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574664.086287:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:0.0:1788574664.086289:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574664.086291:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574664.086292:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:0.0:1788574664.086295:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574664.086296:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086297:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086303:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574664.086304:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086305:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574664.086306:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574664.086307:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574664.086310:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086315:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574664.086316:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574664.086317:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574664.086319:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574664.086320:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574664.086322:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574664.086324:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574664.086326:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574664.086327:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574664.086328:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574664.086330:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574664.086331:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574664.086334:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574664.086336:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574664.086338:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574664.086340:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393151488, portal 10
00000100:00000001:0.0:1788574664.086341:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574664.086342:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:0.0:1788574664.086346:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086351:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574664.086352:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574664.086354:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574664.086355:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393151488, offset 0
00000400:00000200:0.0:1788574664.086358:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574664.086360:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574664.086363:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574664.086365:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574664.086366:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574664.086368:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574664.086370:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574664.086372:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330439:330439:256:4294967295] 192.168.202.110@tcp LPNI seq info [330396:330396:8:4294967295]
00000400:00000200:0.0:1788574664.086375:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574664.086379:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574664.086382:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574664.086384:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574664.086386:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574664.086388:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788574664.086408:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574664.086410:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1788574664.086413:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574664.086414:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574664.086416:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574664.086419:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574664.086421:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574664.086422:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574664.086424:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574664.086426:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086431:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574664.086433:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574689 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086444:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086445:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574664.086450:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574664.086452:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574664.086455:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574664.086457:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306267687
00000100:00000001:0.0:1788574664.086460:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306268601
00000100:00000001:0.0:1788574664.086463:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306267687
00000100:00000001:0.0:1788574664.086465:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306268601
00000100:00000001:0.0:1788574664.086467:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574664.086470:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086471:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574664.086476:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574664/real 1788574664]  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574664.086485:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574664.086486:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574664.086488:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574664.086491:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574664.086493:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574664.086495:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574664.086499:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574664.086509:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086517:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574664.086518:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086520:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574664.086522:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574664.086526:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086533:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574664.086537:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086544:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574664.086547:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086549:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574664.086550:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086552:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574664.086554:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574664.086557:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574664.086561:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574664.086563:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574664.086567:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574664.086575:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574664.086580:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574664.086584:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086606:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574664.086608:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574664.086610:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574664.086612:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574664.086614:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574664.086617:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574664.086621:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574664.086623:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574664.086625:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574664.086626:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574664.086630:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574664.086633:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574664.086637:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574664.086639:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574664.086643:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574664.086646:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393151616, portal 4
00000100:00000001:0.0:1788574664.086648:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574664.086650:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:0.0:1788574664.086656:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086663:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574664.086665:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574664.086667:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574664.086670:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393151616, offset 0
00000400:00000200:0.0:1788574664.086674:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574664.086678:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574664.086682:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574664.086685:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574664.086688:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574664.086691:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574664.086693:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574664.086697:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330440:330440:256:4294967295] 192.168.202.110@tcp LPNI seq info [330397:330397:8:4294967295]
00000400:00000200:0.0:1788574664.086702:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574664.086709:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574664.086714:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574664.086718:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574664.086722:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574664.086725:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788574664.086730:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574664.086733:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1788574664.086737:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574664.086739:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574664.086742:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574664.086746:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574664.086749:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574664.086752:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574664.086754:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574664.086758:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574689 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086767:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574664.086771:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574689 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086779:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086781:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574664.086784:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574664.086787:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574664.086791:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574664.086793:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306267833
00000100:00000001:0.0:1788574664.086796:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306268601
00000100:00000001:0.0:1788574664.086798:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306267833
00000100:00000001:0.0:1788574664.086801:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306268601
00000100:00000001:0.0:1788574664.086804:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574664.086806:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.086808:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574664.086812:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574664/real 1788574664]  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574664.086821:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574664.086823:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574664.086825:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574664.086827:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574664.086829:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574664.086832:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574664.086835:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574664.086845:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086853:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574664.086854:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086856:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574664.086858:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574664.086862:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086870:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574664.086875:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086883:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574664.086885:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086887:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574664.086889:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086891:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574664.086893:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574664.086896:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574664.086900:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574664.086903:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574664.086908:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574664.086917:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574664.086921:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086923:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574664.086927:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393151488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086934:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574664.086936:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574664.086939:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574664.086942:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574664.086943:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574664.086945:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574664.086948:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574664.086950:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:0.0:1788574664.086952:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574664.086954:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574664.086955:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:0.0:1788574664.086958:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574664.086959:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086961:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574664.086964:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393151616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574689 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574664.086972:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574664.086973:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574664.086975:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574664.086978:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574664.086979:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574664.086981:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574664.086983:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574664.086985:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:0.0:1788574664.086988:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574664.086989:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574664.086991:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:0.0:1788574664.086993:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574664.086995:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.086997:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574664.087001:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574664.087002:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574664.087005:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574664.087007:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574665.045515:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306268601, at 4306268608
00000100:00000040:0.0:1788574665.045522:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306268601, at 4306268608
00000100:00000010:1.0:1788574665.045533:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d79c0.
00000100:00000010:0.0:1788574665.045535:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574669.205844:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574669.205854:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574669.205858:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574669.205883:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.205908:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574669.205913:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574669.205924:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574669.205933:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574669.205946:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574669.205966:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393151744:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574669.205978:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574669.205983:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574669.205987:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574669.205991:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574669.206003:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574669.206016:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574669.206027:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574669.206040:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574669.206055:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574669.206070:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393151744, portal 25
00000100:00000001:0.0:1788574669.206078:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574669.206082:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:0.0:1788574669.206096:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206116:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574669.206121:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574669.206132:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574669.206145:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393151744, offset 0
00000400:00000200:0.0:1788574669.206173:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574669.206194:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574669.206208:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574669.206218:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574669.206226:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574669.206238:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574669.206248:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574669.206253:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330441:330441:256:4294967295] 192.168.202.110@tcp LPNI seq info [330398:330398:8:4294967295]
00000400:00000200:0.0:1788574669.206270:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574669.206303:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574669.206320:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574669.206330:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574669.206342:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574669.206354:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788574669.206368:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574669.206380:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788574669.206445:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574669.206450:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574669.206460:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574669.206470:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574669.206477:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574669.206481:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574669.206486:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574669.206491:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206504:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574669.206507:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574694 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206515:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574669.206517:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574669.206523:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574669.206526:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574669.206529:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306273722
00000100:00000010:0.0:1788574669.206534:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574669.206551:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574669.206558:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306273722
00000100:00000010:0.0:1788574669.206560:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ae0.
00000100:00000001:0.0:1788574669.206562:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574669.206565:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574669.206567:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574669.206569:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574669.206573:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574669/real 1788574669]  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574669.206580:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574669.206581:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574669.206583:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574669.206585:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574669.206586:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574669.206588:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574669.206591:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574669.206597:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206605:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574669.206607:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.206608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574669.206610:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574669.206613:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206617:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574669.206620:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206625:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574669.206627:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.206628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574669.206629:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.206631:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574669.206633:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574669.206636:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574669.206638:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574669.206640:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574669.206643:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574669.206648:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393151744:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574669.206652:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.206653:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574669.206655:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393151744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206660:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574669.206662:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574669.206665:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574669.206667:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574669.206669:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574669.206671:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574669.206672:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574669.206674:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788574669.206677:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574669.206679:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574669.206680:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:0.0:1788574669.206683:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574669.206684:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.206686:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.206692:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574669.206693:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574669.206695:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574669.206696:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574669.206697:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574669.206700:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206706:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574669.206707:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574669.206708:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574669.206710:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574669.206711:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574669.206713:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151872:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574669.206715:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574669.206717:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574669.206719:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574669.206720:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574669.206722:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574669.206723:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574669.206726:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574669.206728:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574669.206731:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574669.206733:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393151872, portal 10
00000100:00000001:0.0:1788574669.206735:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574669.206735:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:0.0:1788574669.206740:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206745:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574669.206747:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574669.206748:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574669.206750:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393151872, offset 0
00000400:00000200:0.0:1788574669.206752:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574669.206755:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574669.206758:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574669.206760:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574669.206761:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574669.206763:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574669.206765:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574669.206767:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330442:330442:256:4294967295] 192.168.202.110@tcp LPNI seq info [330399:330399:8:4294967295]
00000400:00000200:0.0:1788574669.206770:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574669.206775:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574669.206778:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574669.206781:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574669.206783:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574669.206786:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788574669.206788:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574669.206790:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788574669.206793:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574669.206795:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574669.206797:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574669.206799:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574669.206801:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574669.206803:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574669.206804:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574669.206806:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206816:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574669.206818:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574694 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206822:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574669.206823:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574669.206825:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574669.206828:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574669.206830:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574669.206831:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306272807
00000100:00000001:0.0:1788574669.206833:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306273722
00000100:00000001:0.0:1788574669.206834:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306272807
00000100:00000001:0.0:1788574669.206836:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306273722
00000100:00000001:0.0:1788574669.206838:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574669.206839:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574669.206840:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574669.206842:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574669/real 1788574669]  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574669.206847:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574669.206848:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574669.206849:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574669.206850:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574669.206851:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574669.206852:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574669.206854:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574669.206859:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206874:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574669.206876:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.206877:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574669.206878:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574669.206884:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206888:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574669.206891:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206896:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574669.206897:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.206898:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574669.206899:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.206900:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574669.206902:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574669.206903:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574669.206905:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574669.206907:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574669.206909:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574669.206914:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393151872:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574669.206917:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574669.206920:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206924:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574669.206925:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574669.206926:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574669.206927:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574669.206929:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574669.206931:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393152000:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574669.206933:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574669.206935:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574669.206936:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574669.206937:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574669.206939:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574669.206941:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574669.206943:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574669.206944:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574669.206946:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574669.206948:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393152000, portal 4
00000100:00000001:0.0:1788574669.206950:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574669.206951:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:0.0:1788574669.206955:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.206959:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574669.206960:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574669.206962:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574669.206964:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393152000, offset 0
00000400:00000200:0.0:1788574669.206966:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574669.206968:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574669.206970:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574669.206972:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574669.206974:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574669.206975:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574669.206977:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574669.206979:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330443:330443:256:4294967295] 192.168.202.110@tcp LPNI seq info [330400:330400:8:4294967295]
00000400:00000200:0.0:1788574669.206982:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574669.206986:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574669.206989:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574669.206991:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574669.206993:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574669.206995:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788574669.206998:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574669.207000:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788574669.207002:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574669.207004:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574669.207006:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574669.207008:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574669.207010:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574669.207011:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574669.207012:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574669.207015:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.207020:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574669.207022:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574694 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.207026:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574669.207027:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574669.207030:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574669.207032:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574669.207036:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574669.207038:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306272953
00000100:00000001:0.0:1788574669.207041:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306273722
00000100:00000001:0.0:1788574669.207043:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306272953
00000100:00000001:0.0:1788574669.207046:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306273722
00000100:00000001:0.0:1788574669.207048:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574669.207049:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574669.207051:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574669.207056:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574669/real 1788574669]  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574669.207065:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574669.207066:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574669.207069:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574669.207071:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574669.207073:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574669.207075:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574669.207079:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574669.207088:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.207093:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574669.207094:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.207095:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574669.207096:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574669.207099:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.207103:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574669.207106:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.207110:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574669.207111:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.207112:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574669.207113:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.207114:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574669.207115:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574669.207117:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574669.207119:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574669.207120:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574669.207123:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574669.207132:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393152000:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574669.207134:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.207136:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574669.207138:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393151872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.207142:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574669.207143:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574669.207144:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574669.207146:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574669.207147:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574669.207148:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574669.207150:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574669.207151:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788574669.207153:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574669.207153:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574669.207155:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:0.0:1788574669.207156:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574669.207157:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.207158:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574669.207160:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393152000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574694 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574669.207164:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574669.207165:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574669.207166:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574669.207168:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574669.207169:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574669.207170:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574669.207171:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574669.207172:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788574669.207173:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574669.207173:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574669.207174:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:0.0:1788574669.207176:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574669.207177:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.207179:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.207181:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574669.207182:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574669.207184:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574669.207185:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574669.717770:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574669.717778:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574669.717781:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574669.717786:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574669.717790:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574669.717793:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.717795:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574669.717797:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574669.717799:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574670.165507:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306273722, at 4306273728
00000100:00000010:1.0:1788574670.165521:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ae0.
00000100:00000040:0.0:1788574670.165543:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306273722, at 4306273728
00000100:00000010:0.0:1788574670.165558:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574670.741478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574670.741491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574670.741493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574670.741501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574670.741505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574670.741509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574670.741512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574670.741515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574670.741518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574671.765495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574671.765501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574671.765502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574671.765507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574671.765535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574671.765539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574671.765541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574671.765543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574671.765545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574672.789479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574672.789486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574672.789489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574672.789495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574672.789498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574672.789512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574672.789514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574672.789516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574672.789519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574673.813601:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574673.813606:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574673.813608:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574673.813612:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574673.813615:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574673.813618:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574673.813620:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574673.813622:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574673.813626:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574674.325519:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.0:1788574674.325523:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000100:00080000:1.0:1788574674.325526:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574674.325530:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.0:1788574674.325532:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000100:00080000:1.0:1788574674.325534:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000800:00000200:0.0:1788574674.325537:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:1.0:1788574674.325539:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574674.325545:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11605
00000400:00000200:1.0:1788574674.325554:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574674.325560:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574674.325562:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574674.325565:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325567:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574674.325569:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325571:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574674.325574:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574674.325576:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574674.325578:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574674.325580:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574674.325582:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574674.325583:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574674.325586:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325587:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574674.325589:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325590:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574674.325592:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574674.325595:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325596:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574674.325599:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574674.325602:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574674.325604:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574674.325605:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574674.325608:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574674.325614:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788574674.325616:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574674.325620:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574674.325621:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574674.325623:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574674.325626:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574674.325629:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574674.325635:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574674.325641:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574674.325649:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393152128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574674.325657:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574674.325660:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574674.325664:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393152128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1788574674.325671:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788574674.325678:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000001:1.0:1788574674.325683:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574674.325687:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000001:0.0:1788574674.325689:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1788574674.325691:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574674.325695:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574674.325696:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000001:0.0:1788574674.325698:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000001:1.0:1788574674.325698:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574674.325703:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11605
00000800:00000001:0.0:1788574674.325705:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:1.0:1788574674.325707:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574674.325713:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574674.325715:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574674.325718:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325720:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574674.325722:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325724:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574674.325726:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574674.325727:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574674.325729:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574674.325731:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574674.325736:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574674.325738:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574674.325744:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325748:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574674.325750:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325753:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574674.325755:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574674.325759:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325763:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574674.325765:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574674.325771:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574674.325775:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574674.325779:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574674.325784:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574674.325790:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574674.325792:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574674.325798:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788574674.325804:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574674.325808:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574674.325809:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574674.325813:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574674.325816:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574674.325819:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574674.325824:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574674.325827:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574674.325835:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393152256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574674.325844:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574674.325847:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574674.325853:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393152256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574674.325864:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574674.325868:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574674.325870:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574674.325873:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574674.325876:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574674.325879:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574674.325881:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11605
00000400:00000200:1.0:1788574674.325886:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574674.325890:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574674.325891:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574674.325894:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325899:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574674.325904:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325907:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574674.325910:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574674.325911:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574674.325913:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574674.325914:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574674.325917:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574674.325919:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574674.325928:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325933:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574674.325938:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325945:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574674.325950:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574674.325969:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574674.325980:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574674.325986:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574674.325991:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574674.325998:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574674.326003:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574674.326006:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574674.326011:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574674.326016:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574674.326021:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788574674.326027:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574674.326033:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574674.326037:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574674.326042:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574674.326044:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574674.326046:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574674.326052:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788574674.326058:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574674.326064:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:1.0:1788574674.326067:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393152384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1788574674.326068:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:1.0:1788574674.326082:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574674.326087:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574674.326093:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393152384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574674.326105:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574674.326109:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574674.326116:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574674.326120:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11615)
00000800:00000200:2.1:1788574674.326449:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574674.326451:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788574674.326714:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574674.326716:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:0.0:1788574674.837522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574674.837529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574674.837532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574674.837538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574674.837542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574674.837545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574674.837548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574674.837551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574674.837554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574675.285505:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306278841, at 4306278848
00000100:00000040:0.0:1788574675.285510:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306278841, at 4306278848
00000100:00000010:1.0:1788574675.285520:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00000100:00000010:0.0:1788574675.285524:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000100:00000001:0.0:1788574675.861550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574675.861557:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574675.861558:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574675.861563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574675.861565:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574675.861569:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574675.861571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574675.861573:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574675.861575:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574676.885545:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574676.885562:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574676.885565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574676.885571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574676.885576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574676.885580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574676.885584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574676.885608:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574676.885613:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574677.909556:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574677.909561:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574677.909563:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574677.909567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574677.909570:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574677.909573:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574677.909576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574677.909577:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574677.909579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574678.933533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574678.933539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574678.933541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574678.933545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574678.933548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574678.933551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574678.933554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574678.933555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574678.933558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574679.446255:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574679.446262:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574679.446920:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574679.446923:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574679.447238:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574679.447241:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1788574679.957539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574679.957545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574679.957547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574679.957552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574679.957555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574679.957559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574679.957561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574679.957563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574679.957565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574680.405549:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306283962, at 4306283968
00000100:00000040:1.0:1788574680.405549:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306283962, at 4306283968
00000100:00000010:0.0:1788574680.405565:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000100:00000010:1.0:1788574680.405569:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000100:00000001:0.0:1788574680.981558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574680.981569:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574680.981572:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574680.981580:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574680.981585:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574680.981590:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574680.981593:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574680.981596:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574680.981599:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574682.005519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574682.005526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574682.005527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574682.005532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574682.005534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574682.005538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574682.005540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574682.005542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574682.005544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574683.029469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574683.029475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574683.029476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574683.029480:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574683.029483:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574683.029493:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574683.029495:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574683.029497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574683.029499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574684.053522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574684.053530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574684.053534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574684.053540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574684.053545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574684.053550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574684.053553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574684.053555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574684.053559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574684.566144:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574684.566146:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574684.566559:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574684.566562:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574684.566848:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574684.566851:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1788574685.077482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574685.077488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574685.077490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574685.077493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574685.077497:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574685.077501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574685.077504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574685.077507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574685.077510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574685.525466:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306289081, at 4306289088
00000100:00000040:1.0:1788574685.525466:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306289081, at 4306289088
00000100:00000010:0.0:1788574685.525481:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000100:00000010:1.0:1788574685.525486:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000100:00000001:0.0:1788574686.101494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574686.101501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574686.101502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574686.101507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574686.101510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574686.101513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574686.101515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574686.101517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574686.101519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574689.685907:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574689.685910:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574689.686236:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574689.686238:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574689.686464:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574689.686466:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:0.0:1788574690.645463:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306294201, at 4306294208
00000100:00000040:1.0:1788574690.645466:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306294201, at 4306294208
00000100:00000010:0.0:1788574690.645478:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000100:00000010:1.0:1788574690.645480:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000100:00000001:0.0:1788574694.805839:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574694.805845:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574694.805849:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574694.805857:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.805869:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574694.805873:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574694.805877:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574694.805881:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574694.805885:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574694.805890:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393153664:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574694.805898:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574694.805903:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574694.805905:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574694.805910:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574694.805916:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574694.805920:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574694.805927:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574694.805932:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574694.805941:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574694.805947:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393153664, portal 25
00000100:00000001:0.0:1788574694.805951:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574694.805953:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:0.0:1788574694.805967:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.805981:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574694.805985:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574694.805988:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574694.805990:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393153664, offset 0
00000400:00000200:0.0:1788574694.805996:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574694.806005:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574694.806013:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574694.806019:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574694.806023:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574694.806028:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574694.806033:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574694.806038:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330456:330456:256:4294967295] 192.168.202.110@tcp LPNI seq info [330413:330413:8:4294967295]
00000400:00000200:0.0:1788574694.806057:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574694.806077:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574694.806093:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574694.806100:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574694.806110:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574694.806115:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1788574694.806125:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574694.806131:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:0.0:1788574694.806141:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574694.806145:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574694.806152:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574694.806159:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574694.806164:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574694.806168:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574694.806173:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574694.806179:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806187:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574694.806191:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574719 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806202:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574694.806206:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574694.806212:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574694.806218:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574694.806223:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306299321
00000100:00000010:0.0:1788574694.806233:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574694.806240:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574694.806246:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306299321
00000100:00000010:0.0:1788574694.806252:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d70c0.
00000100:00000001:0.0:1788574694.806258:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574694.806262:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574694.806268:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574694.806274:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574694.806279:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574694/real 1788574694]  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574694.806310:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574694.806313:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574694.806315:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574694.806319:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574694.806323:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574694.806325:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574694.806327:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574694.806335:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806348:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574694.806351:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.806357:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574694.806360:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574694.806367:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806384:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574694.806436:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806459:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574694.806467:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.806472:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574694.806476:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.806481:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574694.806485:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574694.806491:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574694.806503:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574694.806509:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574694.806516:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574694.806526:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393153664:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574694.806532:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.806534:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574694.806537:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393153664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806545:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574694.806547:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574694.806550:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574694.806553:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574694.806555:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574694.806556:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574694.806557:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574694.806560:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788574694.806562:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574694.806564:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574694.806565:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:0.0:1788574694.806567:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574694.806568:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.806570:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.806576:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574694.806578:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574694.806579:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574694.806580:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574694.806581:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574694.806584:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806589:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574694.806590:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574694.806592:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574694.806593:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574694.806594:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574694.806596:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393153792:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574694.806599:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574694.806601:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574694.806602:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574694.806603:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574694.806605:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574694.806607:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574694.806610:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574694.806612:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574694.806615:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574694.806617:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393153792, portal 10
00000100:00000001:0.0:1788574694.806619:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574694.806620:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:0.0:1788574694.806624:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806628:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574694.806630:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574694.806632:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574694.806633:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393153792, offset 0
00000400:00000200:0.0:1788574694.806636:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574694.806639:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574694.806641:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574694.806643:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574694.806645:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574694.806647:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574694.806649:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574694.806651:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330457:330457:256:4294967295] 192.168.202.110@tcp LPNI seq info [330414:330414:8:4294967295]
00000400:00000200:0.0:1788574694.806654:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574694.806659:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574694.806661:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574694.806664:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574694.806666:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574694.806669:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1788574694.806673:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574694.806676:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:0.0:1788574694.806680:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574694.806682:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574694.806685:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574694.806689:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574694.806692:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574694.806694:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574694.806697:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574694.806700:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806709:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574694.806712:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574719 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806720:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574694.806721:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574694.806725:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574694.806729:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574694.806731:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574694.806733:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306298407
00000100:00000001:0.0:1788574694.806735:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306299321
00000100:00000001:0.0:1788574694.806737:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306298407
00000100:00000001:0.0:1788574694.806738:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306299321
00000100:00000001:0.0:1788574694.806740:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574694.806741:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574694.806743:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574694.806745:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574694/real 1788574694]  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574694.806750:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574694.806751:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574694.806753:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574694.806755:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574694.806756:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574694.806758:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574694.806760:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574694.806765:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806769:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574694.806770:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.806772:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574694.806773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574694.806775:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806779:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574694.806781:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806785:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574694.806787:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.806788:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574694.806788:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.806790:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574694.806791:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574694.806793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574694.806795:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574694.806796:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574694.806798:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574694.806803:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393153792:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574694.806806:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574694.806810:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806817:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574694.806818:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574694.806820:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574694.806822:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574694.806824:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574694.806828:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393153920:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574694.806831:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574694.806834:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574694.806835:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574694.806837:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574694.806841:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574694.806843:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574694.806848:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574694.806850:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574694.806854:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574694.806857:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393153920, portal 4
00000100:00000001:0.0:1788574694.806859:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574694.806860:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:0.0:1788574694.806867:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806874:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574694.806877:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574694.806880:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574694.806882:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393153920, offset 0
00000400:00000200:0.0:1788574694.806887:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574694.806891:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574694.806895:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574694.806898:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574694.806902:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574694.806904:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574694.806907:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574694.806910:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330458:330458:256:4294967295] 192.168.202.110@tcp LPNI seq info [330415:330415:8:4294967295]
00000400:00000200:0.0:1788574694.806915:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574694.806922:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574694.806927:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574694.806930:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574694.806934:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574694.806937:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1788574694.806941:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574694.806944:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:0.0:1788574694.806948:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574694.806949:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574694.806952:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574694.806954:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574694.806956:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574694.806958:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574694.806960:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574694.806962:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806968:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574694.806970:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574719 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.806974:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574694.806976:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574694.806978:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574694.806980:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574694.806982:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574694.806984:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306298553
00000100:00000001:0.0:1788574694.806986:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306299321
00000100:00000001:0.0:1788574694.806987:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306298553
00000100:00000001:0.0:1788574694.806989:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306299321
00000100:00000001:0.0:1788574694.806991:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574694.806992:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574694.806993:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574694.806996:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574694/real 1788574694]  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574694.807001:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574694.807003:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574694.807004:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574694.807005:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574694.807006:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574694.807008:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574694.807010:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574694.807015:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.807019:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574694.807020:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.807021:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574694.807022:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574694.807024:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.807028:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574694.807031:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.807037:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574694.807040:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.807041:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574694.807042:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.807045:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574694.807047:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574694.807050:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574694.807053:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574694.807055:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574694.807061:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574694.807069:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393153920:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574694.807073:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.807076:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574694.807079:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393153792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.807087:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574694.807089:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574694.807091:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574694.807094:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574694.807095:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574694.807097:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574694.807099:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574694.807101:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788574694.807103:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574694.807105:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574694.807106:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:0.0:1788574694.807109:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574694.807111:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.807113:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574694.807117:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393153920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574719 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574694.807124:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574694.807125:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574694.807127:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574694.807130:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574694.807131:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574694.807132:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574694.807134:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574694.807136:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788574694.807138:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574694.807139:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574694.807140:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:0.0:1788574694.807142:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574694.807143:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.807144:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574694.807147:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574694.807148:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574694.807149:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574694.807151:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574695.765541:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306299321, at 4306299328
00000100:00000040:1.0:1788574695.765541:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306299321, at 4306299328
00000100:00000010:1.0:1788574695.765562:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d70c0.
00000100:00000010:0.0:1788574695.765563:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574698.389566:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574698.389576:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574698.389579:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574698.389587:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574698.389593:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574698.389599:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574698.389605:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574698.389610:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574698.389613:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.413522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574699.413528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574699.413529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574699.413533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574699.413536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574699.413540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.413542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574699.413544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.413546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.925727:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574699.925732:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574699.925737:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574699.925747:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.925762:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574699.925764:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574699.925771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574699.925774:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574699.925777:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574699.925784:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393154048:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574699.925792:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574699.925797:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574699.925800:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574699.925805:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574699.925812:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574699.925816:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574699.925827:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574699.925832:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574699.925839:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574699.925848:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393154048, portal 25
00000100:00000001:0.0:1788574699.925853:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574699.925855:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:0.0:1788574699.925865:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.925879:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574699.925884:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574699.925889:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574699.925894:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393154048, offset 0
00000400:00000200:0.0:1788574699.925903:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574699.925915:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574699.925925:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574699.925932:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574699.925954:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574699.925963:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574699.925970:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574699.925979:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330459:330459:256:4294967295] 192.168.202.110@tcp LPNI seq info [330416:330416:8:4294967295]
00000400:00000200:0.0:1788574699.925990:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574699.926008:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574699.926018:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574699.926025:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574699.926036:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574699.926040:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788574699.926048:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574699.926051:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1788574699.926057:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574699.926062:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574699.926069:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574699.926075:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574699.926083:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574699.926086:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574699.926091:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574699.926096:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574699.926116:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574724 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926127:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.926129:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574699.926134:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574699.926143:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574699.926148:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306304441
00000100:00000010:0.0:1788574699.926155:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574699.926161:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574699.926168:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306304441
00000100:00000010:0.0:1788574699.926174:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7660.
00000100:00000001:0.0:1788574699.926179:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574699.926186:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574699.926192:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.926197:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574699.926204:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574699/real 1788574699]  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574699.926219:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574699.926220:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574699.926222:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574699.926224:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574699.926225:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574699.926227:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574699.926230:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574699.926236:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926240:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574699.926241:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926243:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574699.926244:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574699.926247:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926252:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574699.926255:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926259:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574699.926261:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926262:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574699.926263:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926265:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574699.926266:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574699.926268:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574699.926271:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574699.926273:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574699.926276:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574699.926281:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393154048:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574699.926285:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926286:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574699.926288:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393154048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926292:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574699.926294:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574699.926296:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574699.926298:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574699.926299:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574699.926300:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574699.926301:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574699.926303:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574699.926306:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574699.926307:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574699.926308:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:0.0:1788574699.926311:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574699.926312:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926313:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926319:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574699.926320:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.926322:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574699.926323:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574699.926324:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574699.926326:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926331:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574699.926332:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574699.926334:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574699.926336:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574699.926337:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574699.926339:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393154176:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574699.926342:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574699.926343:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574699.926344:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574699.926345:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574699.926347:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574699.926348:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574699.926351:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574699.926353:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574699.926355:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574699.926357:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393154176, portal 10
00000100:00000001:0.0:1788574699.926359:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574699.926360:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:0.0:1788574699.926364:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926369:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574699.926371:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574699.926372:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574699.926373:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393154176, offset 0
00000400:00000200:0.0:1788574699.926376:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574699.926378:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574699.926381:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574699.926383:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574699.926385:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574699.926386:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574699.926388:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574699.926390:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330460:330460:256:4294967295] 192.168.202.110@tcp LPNI seq info [330417:330417:8:4294967295]
00000400:00000200:0.0:1788574699.926426:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574699.926437:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574699.926441:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574699.926444:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574699.926447:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574699.926449:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788574699.926452:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574699.926454:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1788574699.926457:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574699.926458:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574699.926460:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574699.926463:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574699.926465:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574699.926466:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574699.926468:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574699.926471:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926476:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574699.926478:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574724 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926483:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.926484:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574699.926486:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574699.926489:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574699.926491:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574699.926493:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306303527
00000100:00000001:0.0:1788574699.926495:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306304441
00000100:00000001:0.0:1788574699.926497:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306303527
00000100:00000001:0.0:1788574699.926498:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306304441
00000100:00000001:0.0:1788574699.926500:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574699.926501:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.926502:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574699.926505:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574699/real 1788574699]  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574699.926511:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574699.926512:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574699.926513:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574699.926514:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574699.926520:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574699.926521:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574699.926523:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574699.926529:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926534:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574699.926535:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926536:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574699.926537:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574699.926539:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926543:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574699.926545:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926549:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574699.926551:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926552:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574699.926552:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926554:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574699.926555:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574699.926557:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574699.926559:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574699.926560:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574699.926563:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574699.926568:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393154176:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574699.926571:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574699.926573:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926577:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574699.926578:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574699.926579:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574699.926580:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574699.926581:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574699.926583:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393154304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574699.926585:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574699.926586:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574699.926587:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574699.926588:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574699.926590:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574699.926592:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574699.926594:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574699.926596:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574699.926598:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574699.926600:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393154304, portal 4
00000100:00000001:0.0:1788574699.926602:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574699.926603:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:0.0:1788574699.926607:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926611:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574699.926613:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574699.926614:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574699.926615:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393154304, offset 0
00000400:00000200:0.0:1788574699.926618:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574699.926620:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574699.926623:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574699.926625:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574699.926626:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574699.926628:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574699.926630:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574699.926632:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330461:330461:256:4294967295] 192.168.202.110@tcp LPNI seq info [330418:330418:8:4294967295]
00000400:00000200:0.0:1788574699.926635:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574699.926639:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574699.926641:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574699.926643:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574699.926646:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574699.926647:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1788574699.926650:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574699.926658:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:0.0:1788574699.926660:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574699.926661:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574699.926663:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574699.926666:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574699.926668:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574699.926669:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574699.926670:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574699.926673:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926678:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574699.926679:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574724 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926684:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.926685:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574699.926686:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574699.926688:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574699.926690:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574699.926692:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306303673
00000100:00000001:0.0:1788574699.926694:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306304441
00000100:00000001:0.0:1788574699.926695:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306303673
00000100:00000001:0.0:1788574699.926697:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306304441
00000100:00000001:0.0:1788574699.926698:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574699.926699:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.926700:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574699.926702:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574699/real 1788574699]  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574699.926707:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574699.926708:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574699.926709:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574699.926710:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574699.926711:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574699.926713:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574699.926715:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574699.926719:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926723:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574699.926724:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926725:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574699.926726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574699.926728:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926732:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574699.926734:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926738:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574699.926740:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926741:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574699.926742:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926743:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574699.926744:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574699.926746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574699.926748:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574699.926749:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574699.926751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574699.926756:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393154304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574699.926758:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926759:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574699.926761:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393154176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926765:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574699.926766:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574699.926767:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574699.926768:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574699.926769:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574699.926770:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574699.926771:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574699.926773:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788574699.926775:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574699.926775:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574699.926776:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:0.0:1788574699.926778:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574699.926779:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926780:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574699.926782:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393154304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574699.926791:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574699.926792:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574699.926793:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574699.926794:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574699.926795:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574699.926796:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574699.926797:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574699.926798:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788574699.926799:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574699.926800:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574699.926800:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:0.0:1788574699.926802:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574699.926802:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926804:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574699.926806:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574699.926807:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574699.926808:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574699.926809:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574700.437491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574700.437498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574700.437499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574700.437504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574700.437507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574700.437510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574700.437512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574700.437514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574700.437516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574700.885463:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306304441, at 4306304448
00000100:00000040:0.0:1788574700.885465:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306304441, at 4306304448
00000100:00000010:1.0:1788574700.885481:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7660.
00000100:00000010:0.0:1788574700.885483:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574701.461514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574701.461521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574701.461523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574701.461528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574701.461531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574701.461534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574701.461537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574701.461540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574701.461543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574702.485495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574702.485502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574702.485504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574702.485508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574702.485511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574702.485513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574702.485515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574702.485518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574702.485520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.045689:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574705.045694:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574705.045697:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574705.045707:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.045728:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574705.045731:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574705.045736:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574705.045739:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574705.045746:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574705.045752:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393154432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574705.045760:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574705.045765:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574705.045768:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574705.045772:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574705.045779:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574705.045782:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574705.045790:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574705.045795:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574705.045803:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574705.045813:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393154432, portal 25
00000100:00000001:0.0:1788574705.045817:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574705.045821:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:0.0:1788574705.045832:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.045850:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574705.045855:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574705.045860:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574705.045866:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393154432, offset 0
00000400:00000200:0.0:1788574705.045877:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574705.045888:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574705.045896:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574705.045904:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574705.045910:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574705.045915:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574705.045919:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574705.045926:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330462:330462:256:4294967295] 192.168.202.110@tcp LPNI seq info [330419:330419:8:4294967295]
00000400:00000200:0.0:1788574705.045933:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574705.045949:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574705.045971:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574705.045978:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574705.045989:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574705.045995:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574705.046003:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574705.046009:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574705.046017:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574705.046022:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574705.046030:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574705.046037:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574705.046045:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574705.046050:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574705.046059:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574705.046068:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046081:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574705.046085:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046093:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.046095:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574705.046101:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574705.046105:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574705.046108:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306309561
00000100:00000010:0.0:1788574705.046114:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574705.046120:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574705.046126:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306309561
00000100:00000010:0.0:1788574705.046129:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7d80.
00000100:00000001:0.0:1788574705.046133:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574705.046136:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574705.046139:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.046141:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574705.046146:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574705/real 1788574705]  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574705.046154:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574705.046155:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574705.046157:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574705.046159:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574705.046161:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574705.046163:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574705.046165:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574705.046171:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046175:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574705.046176:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046178:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574705.046179:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574705.046182:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046187:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574705.046191:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046195:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574705.046198:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046199:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574705.046199:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046201:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574705.046202:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574705.046204:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574705.046207:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574705.046208:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574705.046211:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574705.046225:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393154432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574705.046229:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046230:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574705.046233:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393154432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046237:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574705.046240:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574705.046242:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574705.046244:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574705.046245:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574705.046246:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574705.046247:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574705.046249:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788574705.046252:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574705.046254:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574705.046255:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:0.0:1788574705.046258:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574705.046259:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046260:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046266:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574705.046267:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.046269:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574705.046270:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574705.046271:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574705.046273:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046278:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574705.046279:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574705.046281:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574705.046282:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574705.046283:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574705.046285:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393154560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574705.046287:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574705.046289:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574705.046290:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574705.046291:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574705.046294:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574705.046295:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574705.046297:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574705.046300:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574705.046302:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574705.046304:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393154560, portal 10
00000100:00000001:0.0:1788574705.046306:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574705.046307:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:0.0:1788574705.046316:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046321:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574705.046323:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574705.046325:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574705.046326:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393154560, offset 0
00000400:00000200:0.0:1788574705.046329:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574705.046332:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574705.046335:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574705.046337:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574705.046339:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574705.046341:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574705.046343:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574705.046345:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330463:330463:256:4294967295] 192.168.202.110@tcp LPNI seq info [330420:330420:8:4294967295]
00000400:00000200:0.0:1788574705.046348:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574705.046354:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574705.046359:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574705.046362:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574705.046366:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574705.046369:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574705.046373:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574705.046376:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574705.046380:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574705.046382:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574705.046385:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574705.046388:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574705.046421:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574705.046423:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574705.046425:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574705.046428:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046434:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574705.046436:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046441:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.046442:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574705.046445:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574705.046448:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574705.046450:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574705.046452:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306308647
00000100:00000001:0.0:1788574705.046454:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306309561
00000100:00000001:0.0:1788574705.046456:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306308647
00000100:00000001:0.0:1788574705.046457:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306309561
00000100:00000001:0.0:1788574705.046458:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574705.046460:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.046461:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574705.046464:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574705/real 1788574705]  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574705.046469:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574705.046469:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574705.046471:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574705.046472:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574705.046473:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574705.046487:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574705.046491:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574705.046497:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046502:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574705.046503:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046504:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574705.046505:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574705.046507:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046511:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574705.046514:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046519:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574705.046521:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046522:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574705.046523:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046524:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574705.046526:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574705.046527:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574705.046529:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574705.046531:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574705.046533:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574705.046538:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393154560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574705.046541:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574705.046543:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046547:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574705.046548:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574705.046550:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574705.046557:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574705.046558:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574705.046560:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393154688:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574705.046563:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574705.046564:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574705.046565:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574705.046566:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574705.046569:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574705.046571:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574705.046573:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574705.046575:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574705.046578:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574705.046580:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393154688, portal 4
00000100:00000001:0.0:1788574705.046581:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574705.046582:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:0.0:1788574705.046586:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046591:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574705.046592:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574705.046594:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574705.046595:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393154688, offset 0
00000400:00000200:0.0:1788574705.046597:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574705.046600:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574705.046603:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574705.046605:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574705.046606:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574705.046608:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574705.046610:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574705.046612:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330464:330464:256:4294967295] 192.168.202.110@tcp LPNI seq info [330421:330421:8:4294967295]
00000400:00000200:0.0:1788574705.046615:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574705.046619:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574705.046622:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574705.046624:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574705.046626:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574705.046628:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788574705.046631:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574705.046633:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788574705.046635:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574705.046636:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574705.046639:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574705.046641:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574705.046643:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574705.046644:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574705.046646:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574705.046649:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046654:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574705.046656:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046660:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.046661:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574705.046663:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574705.046665:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574705.046667:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574705.046668:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306308793
00000100:00000001:0.0:1788574705.046670:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306309561
00000100:00000001:0.0:1788574705.046672:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306308793
00000100:00000001:0.0:1788574705.046673:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306309561
00000100:00000001:0.0:1788574705.046674:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574705.046676:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.046677:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574705.046679:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574705/real 1788574705]  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574705.046683:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574705.046684:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574705.046685:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dad68.
00000400:00000200:0.0:1788574705.046687:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4198
00000400:00000010:0.0:1788574705.046688:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4198.
00000100:00000001:0.0:1788574705.046689:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574705.046692:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574705.046697:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046705:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574705.046706:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046707:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574705.046708:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574705.046710:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046714:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574705.046716:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046721:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574705.046722:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046723:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574705.046724:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046725:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574705.046726:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574705.046727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574705.046729:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574705.046731:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574705.046733:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574705.046738:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393154688:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574705.046740:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046741:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574705.046743:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393154560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046747:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574705.046749:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574705.046751:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574705.046752:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574705.046753:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574705.046754:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574705.046756:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574705.046757:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788574705.046758:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574705.046759:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574705.046760:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:0.0:1788574705.046762:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574705.046763:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046764:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574705.046767:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393154688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574705.046771:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574705.046771:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574705.046772:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574705.046774:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574705.046774:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574705.046775:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574705.046776:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574705.046777:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788574705.046778:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574705.046779:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574705.046779:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:0.0:1788574705.046781:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574705.046782:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046783:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.046785:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574705.046786:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.046787:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574705.046789:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574705.557531:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574705.557539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574705.557542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574705.557547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574705.557551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574705.557555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.557558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574705.557561:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574705.557563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574706.005495:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306309561, at 4306309568
00000100:00000040:1.0:1788574706.005496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306309561, at 4306309568
00000100:00000010:1.0:1788574706.005511:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d80.
00000100:00000010:0.0:1788574706.005513:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574706.581440:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574706.581446:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574706.581448:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574706.581452:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574706.581455:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574706.581457:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574706.581459:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574706.581461:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574706.581463:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574707.605454:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574707.605466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574707.605468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574707.605472:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574707.605475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574707.605478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574707.605480:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574707.605482:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574707.605484:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574708.629551:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574708.629559:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574708.629562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574708.629567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574708.629570:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574708.629574:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574708.629576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574708.629578:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574708.629581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574709.653503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574709.653512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574709.653515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574709.653521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574709.653526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574709.653532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574709.653535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574709.653539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574709.653543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574710.165686:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574710.165692:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574710.165696:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574710.165710:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393154816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574710.165731:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574710.165736:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574710.165745:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574710.165750:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574710.165755:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574710.165764:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393154816:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574710.165772:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574710.165779:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574710.165784:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574710.165791:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574710.165801:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574710.165807:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574710.165818:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574710.165824:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dad68.
00000400:00000010:0.0:1788574710.165834:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4198.
00000100:00000200:0.0:1788574710.165845:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393154816, portal 25
00000100:00000001:0.0:1788574710.165852:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574710.165856:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:0.0:1788574710.165866:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393154816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574710.165879:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574710.165884:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574710.165888:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788574710.165892:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393154816, offset 0
00000400:00000200:0.0:1788574710.165903:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574710.165913:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574710.165923:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574710.165933:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574710.165940:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574710.165945:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574710.165951:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574710.165958:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330465:330465:256:4294967295] 192.168.202.110@tcp LPNI seq info [330422:330422:8:4294967295]
00000400:00000200:0.0:1788574710.165966:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574710.165977:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574710.165986:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574710.165992:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788574710.165999:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574710.166009:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574710.166042:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574710.166049:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574710.166054:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306314681
00000100:00000010:0.0:1788574710.166062:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574710.166070:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574710.166079:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306314681
00000100:00000010:0.0:1788574710.166091:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7840.
00000100:00000001:0.0:1788574710.166095:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574710.166100:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574710.166102:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574710.166105:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574710.166107:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574710.166114:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574710.166115:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788574710.166117:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574710.166119:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574710.166120:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574710.166124:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393154944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574710.166130:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574710.166131:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574710.166133:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574710.166134:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574710.166136:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574710.166138:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393154944:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574710.166140:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574710.166142:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574710.166143:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574710.166145:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574710.166152:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788574710.166154:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574710.166157:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574710.166160:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dac60.
00000400:00000010:0.0:1788574710.166163:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4220.
00000100:00000200:0.0:1788574710.166165:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393154944, portal 10
00000100:00000001:0.0:1788574710.166167:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574710.166167:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000040:0.0:1788574710.166172:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393154944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574710.166177:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574710.166179:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574710.166181:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574710.166183:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393154944, offset 0
00000400:00000200:0.0:1788574710.166186:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574710.166189:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574710.166193:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574710.166197:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574710.166200:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574710.166203:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574710.166206:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574710.166210:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330466:330466:255:4294967295] 192.168.202.110@tcp LPNI seq info [330423:330423:7:4294967295]
00000400:00000200:0.0:1788574710.166215:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574710.166221:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574710.166227:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574710.166231:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9300.
00000800:00000200:0.0:1788574710.166236:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574710.166240:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574710.166244:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574710.166248:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574710.166251:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574710.166253:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306313766
00000100:00000001:0.0:1788574710.166256:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306314681
00000100:00000001:0.0:1788574710.166258:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306313766
00000100:00000001:0.0:1788574710.166260:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306314681
00000100:00000001:0.0:1788574710.166262:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574710.166264:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574710.166266:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574710.166271:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393155072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574710.166279:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574710.166280:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574710.166282:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574710.166284:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574710.166286:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574710.166289:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393155072:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574710.166293:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574710.166295:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574710.166297:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574710.166299:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574710.166302:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788574710.166304:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574710.166307:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574710.166310:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da9f8.
00000400:00000010:0.0:1788574710.166313:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d45d8.
00000100:00000200:0.0:1788574710.166317:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393155072, portal 4
00000100:00000001:0.0:1788574710.166319:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574710.166320:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:0.0:1788574710.166326:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393155072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574710.166333:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574710.166335:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574710.166337:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574710.166340:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393155072, offset 0
00000400:00000200:0.0:1788574710.166344:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574710.166348:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574710.166352:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574710.166355:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574710.166357:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574710.166359:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574710.166362:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574710.166365:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330467:330467:254:4294967295] 192.168.202.110@tcp LPNI seq info [330424:330424:6:4294967295]
00000400:00000200:0.0:1788574710.166369:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574710.166375:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574710.166380:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574710.166384:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574710.166388:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574710.166414:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574710.166423:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574710.166426:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574710.166431:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574710.166433:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306313913
00000100:00000001:0.0:1788574710.166435:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306314681
00000100:00000001:0.0:1788574710.166438:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306313913
00000100:00000001:0.0:1788574710.166440:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306314681
00000100:00000001:0.0:1788574710.166442:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574710.166444:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574710.166447:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574710.166449:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574710.166453:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574710.166454:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788574710.166456:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574710.166458:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574710.166459:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574710.166462:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574710.677518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574710.677524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574710.677526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574710.677530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574710.677533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574710.677536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574710.677538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574710.677541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574710.677543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574711.125478:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306314681, at 4306314688
00000100:00000040:0.0:1788574711.125480:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306314681, at 4306314688
00000100:00000010:1.0:1788574711.125494:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7840.
00000100:00000010:0.0:1788574711.125496:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574711.701475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574711.701484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574711.701486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574711.701491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574711.701495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574711.701499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574711.701502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574711.701505:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574711.701508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574712.725500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574712.725506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574712.725508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574712.725513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574712.725516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574712.725519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574712.725521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574712.725524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574712.725527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574713.237637:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574713.237645:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574713.237649:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000400:00000100:0.0:1788574713.237660:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:0.0:1788574713.237700:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1788574713.237710:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574713.237715:0:2350:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:0.0:1788574713.237722:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574713.237726:0:2350:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1788574713.237731:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788574713.237751:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:0.0:1788574713.237759:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788574713.237765:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000800:00000200:1.0:1788574713.237766:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574713.237773:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000400:00000200:0.0:1788574713.237774:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788574713.237778:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788574713.237784:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000800:00000001:1.0:1788574713.237787:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000200:0.0:1788574713.237793:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e02300 x1875444393154816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000001:1.0:1788574713.237795:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574713.237800:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574713.237818:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574713.237824:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393154816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574713.237837:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574713.237840:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788574713.237851:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788574713.237854:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9300.
00000400:00000200:0.0:1788574713.237857:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788574713.237861:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788574713.237865:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574713.237882:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574713.237888:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574713.237896:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e01180 x1875444393154944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574713.237911:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574713.237919:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393154944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574713.237949:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574713.237954:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788574713.237964:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788574713.237969:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574713.237976:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788574713.237985:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788574713.237990:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574713.237995:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574713.238001:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574713.238009:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e03480 x1875444393155072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574713.238024:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574713.238031:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393155072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574713.238058:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574713.238062:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574713.749487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574713.749494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574713.749495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574713.749500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574713.749503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574713.749506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574713.749508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574713.749526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574713.749529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574714.773493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574714.773499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574714.773501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574714.773505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574714.773507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574714.773510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574714.773512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574714.773514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574714.773517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574715.285530:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574715.285542:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574715.285545:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574715.285561:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574715.285567:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574715.285575:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11646
00000400:00000200:0.0:1788574715.285585:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574715.285592:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574715.285596:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574715.285600:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285603:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574715.285606:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285609:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574715.285614:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574715.285616:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574715.285619:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574715.285623:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574715.285626:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574715.285629:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574715.285632:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285635:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574715.285637:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285640:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574715.285643:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574715.285647:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285650:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574715.285653:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574715.285657:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574715.285660:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574715.285663:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574715.285666:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574715.285673:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788574715.285677:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574715.285681:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574715.285683:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574715.285687:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574715.285690:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574715.285694:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574715.285701:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574715.285709:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574715.285718:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393155200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574715.285729:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574715.285734:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574715.285740:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393155200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574715.285765:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574715.285770:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574715.285774:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574715.285776:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574715.285778:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574715.285781:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574715.285784:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11646
00000400:00000200:0.0:1788574715.285789:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574715.285793:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574715.285795:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574715.285798:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285801:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574715.285803:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285805:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574715.285808:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574715.285810:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574715.285812:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574715.285814:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574715.285816:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574715.285819:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574715.285822:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285824:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574715.285826:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285828:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574715.285831:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574715.285835:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285837:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574715.285840:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574715.285845:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574715.285848:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574715.285850:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574715.285855:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574715.285858:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574715.285861:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574715.285872:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788574715.285875:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574715.285878:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574715.285879:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574715.285881:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574715.285882:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574715.285884:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574715.285890:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788574715.285894:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574715.285901:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393155328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574715.285909:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574715.285913:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574715.285917:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393155328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574715.285927:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574715.285930:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574715.285934:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574715.285936:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574715.285938:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574715.285942:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574715.285948:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11646
00000400:00000200:0.0:1788574715.285956:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574715.285963:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574715.285967:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574715.285974:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285979:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574715.285985:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.285989:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574715.285994:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574715.285996:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574715.286000:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574715.286003:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574715.286007:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574715.286009:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574715.286012:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.286015:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574715.286018:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574715.286023:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574715.286028:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574715.286036:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574715.286042:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574715.286045:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574715.286050:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574715.286056:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574715.286060:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574715.286064:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574715.286070:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574715.286076:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574715.286081:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788574715.286086:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574715.286091:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574715.286094:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574715.286098:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574715.286099:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574715.286101:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574715.286107:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574715.286112:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574715.286120:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393155456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574715.286134:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574715.286137:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574715.286147:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393155456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574715.286164:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574715.286169:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574715.286176:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574715.286182:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11656)
00000800:00000200:2.1:1788574715.286295:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574715.286299:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788574715.286786:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574715.286789:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788574715.287217:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574715.287218:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1788574715.797473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574715.797480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574715.797482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574715.797486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574715.797490:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574715.797494:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574715.797496:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574715.797498:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574715.797500:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574716.245453:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306319802, at 4306319808
00000100:00000040:0.0:1788574716.245478:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306319802, at 4306319808
00000100:00000010:1.0:1788574716.245479:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000100:00000010:0.0:1788574716.245488:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000100:00000001:0.0:1788574716.821482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574716.821490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574716.821492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574716.821498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574716.821502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574716.821506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574716.821508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574716.821512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574716.821515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574717.845507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574717.845513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574717.845516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574717.845521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574717.845526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574717.845530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574717.845533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574717.845536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574717.845539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574718.869488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574718.869494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574718.869496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574718.869500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574718.869503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574718.869506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574718.869508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574718.869510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574718.869512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574719.893512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574719.893519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574719.893521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574719.893526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574719.893529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574719.893533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574719.893535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574719.893537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574719.893539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.405793:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574720.405801:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574720.405809:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574720.405827:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.405855:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574720.405857:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574720.405863:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574720.405875:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574720.405896:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574720.405912:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393155584:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574720.405918:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574720.405926:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574720.405929:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574720.405940:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574720.405957:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574720.405964:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574720.405973:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574720.405982:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574720.405993:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574720.406005:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393155584, portal 25
00000100:00000001:0.0:1788574720.406013:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574720.406018:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:0.0:1788574720.406039:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406068:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574720.406081:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574720.406104:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574720.406114:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393155584, offset 0
00000400:00000200:0.0:1788574720.406129:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574720.406146:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574720.406165:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574720.406176:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574720.406184:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574720.406198:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574720.406212:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574720.406222:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330471:330471:256:4294967295] 192.168.202.110@tcp LPNI seq info [330428:330428:8:4294967295]
00000400:00000200:0.0:1788574720.406235:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574720.406268:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574720.406287:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574720.406300:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574720.406317:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574720.406329:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574720.406344:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574720.406350:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574720.406367:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574720.406377:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574720.406388:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574720.406421:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574720.406426:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574720.406428:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574720.406432:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574720.406438:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406501:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574720.406517:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406534:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.406538:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574720.406544:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574720.406548:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574720.406550:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306324922
00000100:00000010:0.0:1788574720.406555:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574720.406562:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574720.406568:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306324922
00000100:00000010:0.0:1788574720.406570:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7600.
00000100:00000001:0.0:1788574720.406572:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574720.406574:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574720.406576:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.406578:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574720.406582:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574720/real 1788574720]  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574720.406593:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574720.406595:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574720.406598:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574720.406601:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574720.406602:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574720.406605:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574720.406610:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574720.406620:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406628:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574720.406629:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.406632:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574720.406634:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574720.406639:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406644:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574720.406647:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406652:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574720.406654:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.406656:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574720.406657:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.406659:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574720.406661:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574720.406663:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574720.406675:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574720.406677:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574720.406680:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574720.406686:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393155584:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574720.406690:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.406691:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574720.406693:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393155584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406697:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574720.406700:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574720.406702:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574720.406705:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574720.406707:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574720.406708:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574720.406710:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574720.406712:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574720.406715:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574720.406717:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574720.406718:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:0.0:1788574720.406721:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574720.406722:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.406723:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.406729:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574720.406730:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.406732:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574720.406733:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574720.406734:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574720.406737:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406742:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574720.406744:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574720.406745:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574720.406748:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574720.406749:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574720.406751:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393155712:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574720.406754:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574720.406756:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574720.406757:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574720.406758:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574720.406761:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574720.406762:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574720.406765:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574720.406767:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574720.406770:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574720.406773:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393155712, portal 10
00000100:00000001:0.0:1788574720.406774:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574720.406775:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000040:0.0:1788574720.406780:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406785:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574720.406786:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574720.406788:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574720.406789:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393155712, offset 0
00000400:00000200:0.0:1788574720.406792:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574720.406795:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574720.406797:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574720.406800:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574720.406802:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574720.406804:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574720.406806:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574720.406808:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330472:330472:256:4294967295] 192.168.202.110@tcp LPNI seq info [330429:330429:8:4294967295]
00000400:00000200:0.0:1788574720.406811:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574720.406816:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574720.406819:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574720.406821:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574720.406824:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574720.406826:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574720.406830:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574720.406831:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574720.406834:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574720.406835:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574720.406837:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574720.406840:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574720.406842:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574720.406843:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574720.406845:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574720.406847:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406853:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574720.406855:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406859:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.406860:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574720.406862:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574720.406865:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574720.406867:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574720.406868:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306324007
00000100:00000001:0.0:1788574720.406870:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306324922
00000100:00000001:0.0:1788574720.406871:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306324007
00000100:00000001:0.0:1788574720.406873:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306324922
00000100:00000001:0.0:1788574720.406875:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574720.406876:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.406877:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574720.406880:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574720/real 1788574720]  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574720.406885:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574720.406886:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574720.406887:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574720.406888:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574720.406889:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574720.406890:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574720.406892:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574720.406898:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406902:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574720.406903:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.406904:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574720.406905:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574720.406907:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406911:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574720.406914:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406918:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574720.406919:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.406920:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574720.406921:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.406922:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574720.406924:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574720.406926:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574720.406929:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574720.406930:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574720.406933:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574720.406938:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393155712:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574720.406940:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574720.406943:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406947:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574720.406948:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574720.406949:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574720.406950:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574720.406951:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574720.406953:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393155840:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574720.406955:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574720.406957:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574720.406957:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574720.406958:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574720.406960:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574720.406961:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574720.406964:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574720.406965:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574720.406967:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574720.406969:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393155840, portal 4
00000100:00000001:0.0:1788574720.406970:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574720.406971:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:0.0:1788574720.406975:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.406984:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574720.406985:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574720.406987:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574720.406988:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393155840, offset 0
00000400:00000200:0.0:1788574720.406991:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574720.406993:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574720.406995:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574720.406996:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574720.406998:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574720.407000:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574720.407001:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574720.407003:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330473:330473:256:4294967295] 192.168.202.110@tcp LPNI seq info [330430:330430:8:4294967295]
00000400:00000200:0.0:1788574720.407006:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574720.407010:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574720.407013:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574720.407015:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574720.407017:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574720.407019:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788574720.407022:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574720.407024:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788574720.407026:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574720.407027:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574720.407030:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574720.407032:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574720.407033:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574720.407035:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574720.407036:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574720.407038:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.407043:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574720.407045:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.407054:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.407055:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574720.407056:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574720.407059:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574720.407061:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574720.407062:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306324153
00000100:00000001:0.0:1788574720.407064:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306324922
00000100:00000001:0.0:1788574720.407065:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306324153
00000100:00000001:0.0:1788574720.407066:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306324922
00000100:00000001:0.0:1788574720.407068:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574720.407069:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.407070:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574720.407072:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574720/real 1788574720]  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574720.407076:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574720.407077:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574720.407078:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574720.407079:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574720.407081:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574720.407082:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574720.407084:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574720.407107:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.407111:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574720.407113:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.407114:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574720.407115:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574720.407118:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.407123:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574720.407125:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.407130:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574720.407131:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.407132:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574720.407133:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.407134:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574720.407135:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574720.407137:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574720.407139:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574720.407140:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574720.407143:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574720.407147:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393155840:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574720.407150:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.407151:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574720.407153:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393155712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.407157:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574720.407158:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574720.407160:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574720.407161:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574720.407162:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574720.407163:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574720.407164:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574720.407165:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788574720.407168:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574720.407169:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574720.407170:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:0.0:1788574720.407172:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574720.407173:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.407174:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574720.407176:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393155840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574720.407180:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574720.407181:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574720.407182:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574720.407183:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574720.407184:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574720.407184:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574720.407185:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574720.407186:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788574720.407188:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574720.407188:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574720.407189:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:0.0:1788574720.407191:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574720.407191:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.407193:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.407195:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574720.407196:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.407197:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574720.407198:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574720.917495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574720.917501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574720.917503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574720.917508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574720.917511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574720.917514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.917516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574720.917518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574720.917520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574721.365482:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306324922, at 4306324928
00000100:00000040:1.0:1788574721.365482:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306324922, at 4306324928
00000100:00000010:0.0:1788574721.365497:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574721.365499:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7600.
00000100:00000001:0.0:1788574721.941476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574721.941486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574721.941489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574721.941496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574721.941500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574721.941504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574721.941506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574721.941509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574721.941512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574722.965521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574722.965530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574722.965532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574722.965538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574722.965542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574722.965546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574722.965549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574722.965551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574722.965554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574723.989500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574723.989512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574723.989516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574723.989524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574723.989529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574723.989534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574723.989538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574723.989541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574723.989545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.013520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574725.013528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574725.013530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574725.013537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574725.013541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574725.013545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.013548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574725.013550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.013553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.525850:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574725.525862:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574725.525870:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574725.525886:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.525911:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574725.525918:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574725.525928:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574725.525932:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574725.525937:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574725.525944:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393155968:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574725.525956:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574725.525963:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574725.525966:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574725.525973:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574725.525984:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574725.525992:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574725.526004:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574725.526011:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574725.526023:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574725.526030:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393155968, portal 25
00000100:00000001:0.0:1788574725.526037:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574725.526041:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:0.0:1788574725.526051:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526069:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574725.526079:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574725.526098:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574725.526107:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393155968, offset 0
00000400:00000200:0.0:1788574725.526137:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574725.526161:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574725.526176:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574725.526185:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574725.526192:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574725.526197:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574725.526205:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574725.526212:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330474:330474:256:4294967295] 192.168.202.110@tcp LPNI seq info [330431:330431:8:4294967295]
00000400:00000200:0.0:1788574725.526230:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574725.526251:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574725.526262:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574725.526269:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574725.526282:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574725.526294:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574725.526317:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574725.526329:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574725.526350:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574725.526364:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574725.526380:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574725.526430:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574725.526438:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574725.526443:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574725.526455:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574725.526462:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526492:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574725.526500:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526520:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.526529:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574725.526538:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574725.526552:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574725.526558:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306330042
00000100:00000010:0.0:1788574725.526574:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574725.526588:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574725.526601:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306330042
00000100:00000010:0.0:1788574725.526607:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7f00.
00000100:00000001:0.0:1788574725.526612:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574725.526617:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574725.526622:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.526643:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574725.526650:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574725/real 1788574725]  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574725.526662:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574725.526664:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574725.526667:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574725.526670:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574725.526672:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574725.526675:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574725.526678:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574725.526684:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526690:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574725.526692:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.526694:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574725.526697:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574725.526702:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526709:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574725.526715:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526724:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574725.526727:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.526729:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574725.526730:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.526733:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574725.526736:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574725.526740:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574725.526744:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574725.526746:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574725.526749:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574725.526755:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393155968:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574725.526758:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.526760:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574725.526762:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393155968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526766:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574725.526768:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574725.526771:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574725.526773:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574725.526782:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574725.526784:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574725.526787:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574725.526788:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788574725.526792:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574725.526793:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574725.526795:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:0.0:1788574725.526797:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574725.526798:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.526800:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.526806:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574725.526807:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.526809:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574725.526810:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574725.526812:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574725.526814:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526820:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574725.526821:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574725.526823:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574725.526825:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574725.526827:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574725.526828:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156096:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574725.526831:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574725.526833:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574725.526834:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574725.526835:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574725.526838:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574725.526839:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574725.526842:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574725.526845:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574725.526847:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574725.526850:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393156096, portal 10
00000100:00000001:0.0:1788574725.526851:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574725.526852:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:0.0:1788574725.526857:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526862:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574725.526864:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574725.526865:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574725.526867:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393156096, offset 0
00000400:00000200:0.0:1788574725.526870:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574725.526872:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574725.526875:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574725.526878:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574725.526880:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574725.526882:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574725.526883:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574725.526886:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330475:330475:256:4294967295] 192.168.202.110@tcp LPNI seq info [330432:330432:8:4294967295]
00000400:00000200:0.0:1788574725.526888:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574725.526893:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574725.526897:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574725.526899:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574725.526902:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574725.526904:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574725.526908:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574725.526910:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574725.526913:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574725.526915:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574725.526917:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574725.526920:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574725.526922:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574725.526923:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574725.526925:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574725.526927:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526932:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574725.526934:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526939:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.526940:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574725.526942:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574725.526945:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574725.526947:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574725.526948:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306329127
00000100:00000001:0.0:1788574725.526950:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306330042
00000100:00000001:0.0:1788574725.526952:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306329127
00000100:00000001:0.0:1788574725.526953:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306330042
00000100:00000001:0.0:1788574725.526954:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574725.526956:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.526957:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574725.526959:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574725/real 1788574725]  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574725.526966:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574725.526966:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574725.526968:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574725.526969:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574725.526970:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574725.526971:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574725.526973:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574725.526978:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526982:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574725.526983:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.526985:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574725.526986:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574725.526988:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526992:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574725.526994:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.526998:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574725.527000:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.527001:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574725.527002:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.527003:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574725.527005:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574725.527006:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574725.527008:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574725.527010:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574725.527012:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574725.527018:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156096:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574725.527020:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574725.527023:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.527027:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574725.527028:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574725.527030:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574725.527031:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574725.527032:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574725.527033:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574725.527035:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574725.527037:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574725.527038:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574725.527039:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574725.527041:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574725.527042:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574725.527044:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574725.527046:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574725.527048:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574725.527050:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393156224, portal 4
00000100:00000001:0.0:1788574725.527051:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574725.527052:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:0.0:1788574725.527056:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.527060:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574725.527062:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574725.527063:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574725.527065:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393156224, offset 0
00000400:00000200:0.0:1788574725.527067:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574725.527069:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574725.527072:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574725.527074:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574725.527076:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574725.527079:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574725.527081:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574725.527084:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330476:330476:256:4294967295] 192.168.202.110@tcp LPNI seq info [330433:330433:8:4294967295]
00000400:00000200:0.0:1788574725.527089:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574725.527095:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574725.527099:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574725.527103:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574725.527107:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574725.527109:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788574725.527114:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574725.527116:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788574725.527120:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574725.527122:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574725.527125:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574725.527129:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574725.527133:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574725.527134:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574725.527137:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574725.527141:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.527149:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574725.527152:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.527159:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.527160:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574725.527163:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574725.527166:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574725.527170:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574725.527172:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306329273
00000100:00000001:0.0:1788574725.527174:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306330042
00000100:00000001:0.0:1788574725.527176:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306329273
00000100:00000001:0.0:1788574725.527178:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306330042
00000100:00000001:0.0:1788574725.527180:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574725.527182:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.527184:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574725.527188:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574725/real 1788574725]  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574725.527196:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574725.527198:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574725.527199:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574725.527202:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574725.527204:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574725.527206:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574725.527210:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574725.527219:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.527223:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574725.527225:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.527226:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574725.527227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574725.527229:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.527234:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574725.527237:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.527241:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574725.527243:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.527244:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574725.527245:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.527246:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574725.527248:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574725.527250:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574725.527252:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574725.527253:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574725.527255:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574725.527260:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574725.527263:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.527264:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574725.527266:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393156096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.527270:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574725.527272:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574725.527273:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574725.527275:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574725.527276:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574725.527276:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574725.527278:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574725.527279:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788574725.527281:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574725.527282:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574725.527283:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:0.0:1788574725.527285:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574725.527286:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.527287:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574725.527289:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393156224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574725.527298:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574725.527299:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574725.527301:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574725.527303:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574725.527304:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574725.527306:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574725.527308:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574725.527310:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788574725.527312:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574725.527313:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574725.527315:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:0.0:1788574725.527316:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574725.527318:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.527320:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574725.527324:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574725.527325:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574725.527327:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574725.527329:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574726.037460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574726.037468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574726.037470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574726.037475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574726.037480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574726.037484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574726.037486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574726.037489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574726.037493:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574726.485467:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306330042, at 4306330048
00000100:00000040:1.0:1788574726.485469:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306330042, at 4306330048
00000100:00000010:1.0:1788574726.485484:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f00.
00000100:00000010:0.0:1788574726.485491:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574727.061511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574727.061527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574727.061530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574727.061540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574727.061545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574727.061549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574727.061553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574727.061557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574727.061560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574728.085465:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574728.085471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574728.085473:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574728.085477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574728.085479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574728.085482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574728.085484:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574728.085486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574728.085489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574729.109512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574729.109524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574729.109528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574729.109536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574729.109541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574729.109547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574729.109549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574729.109552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574729.109555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.133539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574730.133548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574730.133550:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574730.133556:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574730.133559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574730.133564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.133567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574730.133570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.133572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574730.645525:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574730.645537:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574730.645540:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574730.645544:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574730.645548:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574730.645556:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11661
00000400:00000200:1.0:1788574730.645566:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574730.645573:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574730.645576:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574730.645579:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574730.645582:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574730.645584:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574730.645586:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574730.645590:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574730.645592:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574730.645594:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574730.645597:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574730.645599:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574730.645600:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574730.645603:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574730.645604:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574730.645606:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574730.645607:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574730.645610:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574730.645613:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574730.645615:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574730.645617:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574730.645621:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574730.645623:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574730.645625:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574730.645627:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574730.645638:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788574730.645641:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574730.645645:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574730.645647:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574730.645650:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574730.645661:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574730.645666:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574730.645674:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574730.645682:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574730.645691:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574730.645699:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574730.645703:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574730.645707:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574730.645729:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574730.645732:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574730.645735:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574730.645736:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574730.645737:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574730.645739:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574730.645741:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11661
00000400:00000200:1.0:1788574730.645743:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574730.645746:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574730.645748:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574730.645750:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574730.645753:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574730.645754:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574730.645757:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574730.645758:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574730.645759:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574730.645760:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788574730.645763:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:1.0:1788574730.645764:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574730.645767:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574730.645771:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788574730.645772:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1788574730.645775:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574730.645776:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1788574730.645782:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574730.645785:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574730.645786:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574730.645793:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:0.0:1788574730.645795:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788574730.645800:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574730.645807:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574730.645809:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574730.645814:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574730.645819:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1788574730.645819:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.645824:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574730.645824:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1788574730.645831:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:1.0:1788574730.645831:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788574730.645836:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:1.0:1788574730.645837:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574730.645844:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574730.645845:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574730.645865:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000100:00100000:0.0:1788574730.645866:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393156352:192.168.202.110@tcp:250:kworker.0
00000020:00000040:1.0:1788574730.645877:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:0.0:1788574730.645879:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788574730.645881:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574730.645883:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574730.645885:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1788574730.645885:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574730.645892:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574730.645893:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574730.645902:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574730.645903:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1788574730.645912:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
02000000:00000010:0.0:1788574730.645917:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
00000100:00000001:1.0:1788574730.645918:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574730.645930:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574730.645936:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788574730.645951:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574730.645951:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574730.645957:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788574730.645961:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000100:00000040:1.0:1788574730.645964:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788574730.645969:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574730.645981:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393156352, portal 25
00000100:00000001:1.0:1788574730.645984:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.645985:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:1.0:1788574730.645991:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1788574730.645992:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345478400 : -130483364073216 : ffff8953796f7100)
00000100:00000040:0.0:1788574730.646002:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788574730.646005:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574730.646011:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574730.646015:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574730.646026:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:0.0:1788574730.646027:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:1.0:1788574730.646032:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11661
00000100:00000040:0.0:1788574730.646034:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574730.646041:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000400:00000200:1.0:1788574730.646042:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:0.0:1788574730.646051:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393156352, offset 0
00000400:00000200:1.0:1788574730.646052:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574730.646058:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1788574730.646067:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788574730.646072:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574730.646075:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574730.646078:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574730.646085:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574730.646087:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788574730.646094:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:1.0:1788574730.646094:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574730.646101:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:0.0:1788574730.646104:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:1.0:1788574730.646107:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1788574730.646111:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:1.0:1788574730.646113:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:0.0:1788574730.646118:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:1.0:1788574730.646123:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788574730.646124:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574730.646133:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330477:330477:256:4294967295] 192.168.202.110@tcp LPNI seq info [330434:330434:8:4294967295]
00000020:00000001:1.0:1788574730.646133:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574730.646149:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788574730.646150:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788574730.646159:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574730.646179:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574730.646182:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574730.646185:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574730.646187:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574730.646189:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574730.646191:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574730.646195:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1788574730.646205:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1788574730.646206:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574730.646213:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574730.646217:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574730.646222:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788574730.646227:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788574730.646227:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574730.646238:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000800:00000010:0.0:1788574730.646242:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000020:00000040:1.0:1788574730.646243:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574730.646250:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574730.646253:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574730.646254:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574730.646258:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574730.646263:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788574730.646264:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:1.0:1788574730.646269:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000800:00000200:0.0:1788574730.646272:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:1.0:1788574730.646274:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574730.646283:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1788574730.646291:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574730.646302:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:1.0:1788574730.646303:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574730.646309:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1788574730.646315:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788574730.646317:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:0.0:1788574730.646320:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574730.646331:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1788574730.646331:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574730.646336:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788574730.646345:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:1.0:1788574730.646350:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:0.0:1788574730.646353:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000100:00000040:1.0:1788574730.646356:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11671)
00000400:00000010:0.0:1788574730.646357:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574730.646368:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574730.646374:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646383:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574730.646386:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646419:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.646420:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574730.646425:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574730.646428:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574730.646431:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306335162
00000100:00000010:0.0:1788574730.646438:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574730.646444:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574730.646451:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306335162
00000100:00000010:0.0:1788574730.646455:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7900.
00000100:00000001:0.0:1788574730.646459:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574730.646462:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574730.646465:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.646468:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574730.646473:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574730/real 1788574730]  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574730.646487:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574730.646488:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574730.646490:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574730.646492:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574730.646494:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574730.646500:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574730.646502:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574730.646508:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646513:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574730.646514:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646516:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574730.646518:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574730.646520:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646525:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574730.646529:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646534:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574730.646536:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646537:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574730.646538:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646539:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574730.646541:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574730.646544:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574730.646547:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574730.646548:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574730.646551:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574730.646557:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393156352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574730.646561:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646563:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574730.646565:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7100 x1875444393156352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646569:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574730.646571:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574730.646573:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574730.646575:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574730.646577:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574730.646578:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574730.646579:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574730.646581:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:0.0:1788574730.646584:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574730.646585:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574730.646586:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7100.
00000100:00000001:0.0:1788574730.646589:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574730.646590:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646592:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646598:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574730.646599:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.646601:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574730.646603:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574730.646604:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574730.646607:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646612:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574730.646613:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574730.646615:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574730.646617:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574730.646618:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574730.646620:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574730.646622:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574730.646624:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574730.646626:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574730.646627:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574730.646629:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574730.646630:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574730.646633:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574730.646635:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574730.646638:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574730.646640:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393156480, portal 10
00000100:00000001:0.0:1788574730.646641:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574730.646642:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345481088 : -130483364070528 : ffff8953796f7b80)
00000100:00000040:0.0:1788574730.646647:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646652:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574730.646653:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574730.646655:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574730.646656:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393156480, offset 0
00000400:00000200:0.0:1788574730.646660:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574730.646664:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574730.646668:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574730.646671:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574730.646674:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574730.646676:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574730.646679:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574730.646682:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330478:330478:256:4294967295] 192.168.202.110@tcp LPNI seq info [330435:330435:8:4294967295]
00000400:00000200:0.0:1788574730.646687:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574730.646694:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574730.646699:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574730.646703:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574730.646707:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574730.646710:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574730.646715:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574730.646717:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574730.646720:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574730.646722:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574730.646724:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574730.646727:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574730.646729:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574730.646731:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574730.646732:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574730.646735:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646741:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574730.646743:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646747:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.646748:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574730.646751:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574730.646753:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574730.646756:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574730.646757:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306334247
00000100:00000001:0.0:1788574730.646759:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306335162
00000100:00000001:0.0:1788574730.646761:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306334247
00000100:00000001:0.0:1788574730.646762:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306335162
00000100:00000001:0.0:1788574730.646764:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574730.646765:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.646766:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574730.646769:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574730/real 1788574730]  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574730.646774:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574730.646775:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574730.646777:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574730.646779:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574730.646780:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574730.646781:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574730.646783:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574730.646788:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646792:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574730.646793:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646794:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574730.646795:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574730.646797:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646801:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574730.646804:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646808:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574730.646810:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646811:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574730.646812:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646813:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574730.646814:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574730.646816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574730.646818:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574730.646820:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574730.646822:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574730.646827:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574730.646830:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574730.646833:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646837:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574730.646838:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574730.646840:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574730.646841:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574730.646842:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574730.646844:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156608:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574730.646846:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574730.646848:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574730.646849:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574730.646850:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574730.646853:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574730.646854:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574730.646857:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574730.646858:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574730.646861:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574730.646863:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393156608, portal 4
00000100:00000001:0.0:1788574730.646864:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574730.646865:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345469440 : -130483364082176 : ffff8953796f4e00)
00000100:00000040:0.0:1788574730.646869:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646874:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574730.646875:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574730.646877:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574730.646878:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393156608, offset 0
00000400:00000200:0.0:1788574730.646881:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574730.646883:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574730.646886:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574730.646888:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574730.646890:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574730.646892:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574730.646893:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574730.646895:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330479:330479:256:4294967295] 192.168.202.110@tcp LPNI seq info [330436:330436:8:4294967295]
00000400:00000200:0.0:1788574730.646898:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574730.646903:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574730.646905:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574730.646907:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574730.646910:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574730.646912:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788574730.646915:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574730.646917:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788574730.646919:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574730.646920:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574730.646922:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574730.646925:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574730.646926:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574730.646928:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574730.646929:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574730.646931:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646936:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574730.646938:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646942:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.646943:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574730.646945:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574730.646947:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574730.646949:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574730.646950:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306334393
00000100:00000001:0.0:1788574730.646952:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306335162
00000100:00000001:0.0:1788574730.646953:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306334393
00000100:00000001:0.0:1788574730.646955:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306335162
00000100:00000001:0.0:1788574730.646957:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574730.646958:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.646959:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574730.646962:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574730/real 1788574730]  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574730.646966:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574730.646967:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574730.646968:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574730.646969:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574730.646970:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574730.646971:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574730.646973:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574730.646979:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646983:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574730.646984:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.646985:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574730.646985:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574730.646988:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646991:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574730.646994:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.646998:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574730.646999:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.647000:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574730.647001:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.647002:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574730.647004:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574730.647005:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574730.647008:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574730.647009:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574730.647011:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574730.647016:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156608:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574730.647018:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.647019:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574730.647021:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7b80 x1875444393156480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.647025:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574730.647026:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574730.647027:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574730.647029:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574730.647030:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574730.647031:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574730.647032:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574730.647033:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:0.0:1788574730.647035:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574730.647036:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574730.647037:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7b80.
00000100:00000001:0.0:1788574730.647039:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574730.647040:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.647041:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574730.647043:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4e00 x1875444393156608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574730.647047:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574730.647047:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574730.647048:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788574730.647050:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574730.647051:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574730.647052:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574730.647054:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574730.647055:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:0.0:1788574730.647056:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574730.647057:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574730.647058:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4e00.
00000100:00000001:0.0:1788574730.647059:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574730.647060:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.647061:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574730.647063:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574730.647064:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574730.647065:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574730.647067:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574731.157565:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574731.157575:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574731.157578:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574731.157585:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574731.157589:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574731.157594:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574731.157598:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574731.157602:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574731.157606:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574731.605512:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306335162, at 4306335168
00000100:00000040:1.0:1788574731.605513:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306335162, at 4306335168
00000100:00000010:0.0:1788574731.605534:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000010:1.0:1788574731.605539:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7900.
00000100:00000001:0.0:1788574732.181490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574732.181496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574732.181498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574732.181502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574732.181504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574732.181507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574732.181509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574732.181511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574732.181514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574733.205503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574733.205515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574733.205519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574733.205527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574733.205532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574733.205537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574733.205539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574733.205542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574733.205544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574734.229521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574734.229528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574734.229531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574734.229538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574734.229543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574734.229547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574734.229550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574734.229553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574734.229557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.253514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574735.253527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574735.253530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574735.253538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574735.253543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574735.253548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.253552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574735.253555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.253560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.765841:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574735.765848:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574735.765853:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574735.765863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.765877:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574735.765880:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574735.765886:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574735.765890:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574735.765896:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574735.765903:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393156736:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574735.765914:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574735.765919:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574735.765923:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574735.765926:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574735.765934:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574735.765938:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574735.765948:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574735.765956:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574735.765965:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574735.765975:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393156736, portal 25
00000100:00000001:0.0:1788574735.765979:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574735.765982:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:0.0:1788574735.765992:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766011:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574735.766015:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574735.766017:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574735.766020:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393156736, offset 0
00000400:00000200:0.0:1788574735.766030:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574735.766041:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574735.766054:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574735.766062:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574735.766068:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574735.766075:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574735.766079:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574735.766085:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330480:330480:256:4294967295] 192.168.202.110@tcp LPNI seq info [330437:330437:8:4294967295]
00000400:00000200:0.0:1788574735.766093:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574735.766106:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574735.766118:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574735.766128:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574735.766137:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574735.766144:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788574735.766157:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574735.766163:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788574735.766173:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574735.766177:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574735.766187:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574735.766202:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574735.766208:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574735.766211:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574735.766216:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574735.766220:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766228:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574735.766231:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766237:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.766238:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574735.766243:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574735.766247:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574735.766250:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306340281
00000100:00000010:0.0:1788574735.766255:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574735.766261:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574735.766267:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306340281
00000100:00000010:0.0:1788574735.766270:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7a80.
00000100:00000001:0.0:1788574735.766272:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574735.766275:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574735.766277:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.766279:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574735.766283:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574735/real 1788574735]  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574735.766291:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574735.766293:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574735.766295:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574735.766297:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574735.766298:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574735.766300:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574735.766303:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574735.766309:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766314:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574735.766315:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766317:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574735.766320:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574735.766324:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766332:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574735.766337:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766345:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574735.766349:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766351:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574735.766352:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766355:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574735.766358:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574735.766361:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574735.766366:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574735.766369:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574735.766374:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574735.766384:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393156736:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574735.766420:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766422:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574735.766425:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393156736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766430:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574735.766433:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574735.766436:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574735.766438:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574735.766440:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574735.766441:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574735.766443:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574735.766446:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:0.0:1788574735.766449:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574735.766451:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574735.766452:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:0.0:1788574735.766455:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574735.766456:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766458:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766465:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574735.766467:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.766469:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574735.766470:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574735.766471:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574735.766474:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766482:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574735.766483:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574735.766485:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574735.766487:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574735.766489:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574735.766493:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156864:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574735.766496:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574735.766499:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574735.766500:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574735.766503:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574735.766506:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574735.766509:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574735.766514:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574735.766517:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574735.766521:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574735.766525:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393156864, portal 10
00000100:00000001:0.0:1788574735.766527:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574735.766529:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:0.0:1788574735.766535:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766540:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574735.766541:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574735.766543:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574735.766545:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393156864, offset 0
00000400:00000200:0.0:1788574735.766548:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574735.766551:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574735.766553:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574735.766556:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574735.766558:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574735.766559:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574735.766561:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574735.766563:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330481:330481:256:4294967295] 192.168.202.110@tcp LPNI seq info [330438:330438:8:4294967295]
00000400:00000200:0.0:1788574735.766566:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574735.766571:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574735.766574:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574735.766577:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574735.766580:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574735.766582:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788574735.766585:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574735.766587:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788574735.766589:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574735.766591:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574735.766593:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574735.766596:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574735.766598:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574735.766599:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574735.766601:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574735.766604:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766610:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574735.766612:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766617:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.766618:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574735.766620:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574735.766623:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574735.766625:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574735.766627:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306339367
00000100:00000001:0.0:1788574735.766629:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306340281
00000100:00000001:0.0:1788574735.766631:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306339367
00000100:00000001:0.0:1788574735.766632:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306340281
00000100:00000001:0.0:1788574735.766634:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574735.766635:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.766636:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574735.766639:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574735/real 1788574735]  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574735.766644:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574735.766645:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574735.766647:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574735.766649:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574735.766650:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574735.766652:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574735.766656:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574735.766665:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766672:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574735.766674:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766676:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574735.766678:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574735.766682:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766714:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574735.766718:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766724:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574735.766726:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766727:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574735.766728:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766729:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574735.766731:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574735.766733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574735.766735:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574735.766737:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574735.766740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574735.766745:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156864:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574735.766748:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574735.766750:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766759:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574735.766760:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574735.766762:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574735.766763:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574735.766765:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574735.766766:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156992:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574735.766768:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574735.766770:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574735.766771:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574735.766772:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574735.766775:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574735.766776:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574735.766780:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574735.766781:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574735.766784:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574735.766787:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393156992, portal 4
00000100:00000001:0.0:1788574735.766788:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574735.766789:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:0.0:1788574735.766793:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766798:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574735.766799:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574735.766801:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574735.766802:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393156992, offset 0
00000400:00000200:0.0:1788574735.766805:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574735.766807:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574735.766810:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574735.766812:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574735.766814:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574735.766816:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574735.766818:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574735.766820:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330482:330482:256:4294967295] 192.168.202.110@tcp LPNI seq info [330439:330439:8:4294967295]
00000400:00000200:0.0:1788574735.766823:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574735.766829:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574735.766832:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574735.766834:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574735.766837:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574735.766839:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788574735.766842:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574735.766844:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788574735.766846:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574735.766848:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574735.766850:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574735.766852:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574735.766855:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574735.766856:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574735.766857:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574735.766860:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766865:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574735.766867:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766872:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.766873:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574735.766875:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574735.766877:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574735.766879:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574735.766880:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306339513
00000100:00000001:0.0:1788574735.766882:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306340281
00000100:00000001:0.0:1788574735.766884:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306339513
00000100:00000001:0.0:1788574735.766886:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306340281
00000100:00000001:0.0:1788574735.766887:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574735.766889:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.766891:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574735.766893:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574735/real 1788574735]  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574735.766898:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574735.766899:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574735.766900:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574735.766902:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574735.766903:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574735.766904:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574735.766906:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574735.766912:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766916:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574735.766917:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766918:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574735.766919:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574735.766921:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766925:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574735.766927:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766931:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574735.766932:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766933:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574735.766934:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766936:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574735.766937:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574735.766939:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574735.766941:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574735.766943:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574735.766946:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574735.766951:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393156992:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574735.766953:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766954:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574735.766956:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393156864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766960:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574735.766961:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574735.766962:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574735.766963:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574735.766965:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574735.766966:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574735.766967:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574735.766968:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:0.0:1788574735.766970:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574735.766971:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574735.766972:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:0.0:1788574735.766974:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574735.766975:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766976:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574735.766978:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393156992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574735.766982:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574735.766983:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574735.766984:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574735.766985:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574735.766986:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574735.766987:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574735.766988:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574735.766989:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:0.0:1788574735.766990:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574735.766991:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574735.766992:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788574735.766995:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574735.766996:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766997:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574735.766999:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574735.767000:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574735.767001:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574735.767002:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574736.277476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574736.277487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574736.277490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574736.277497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574736.277503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574736.277508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574736.277512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574736.277515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574736.277519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574736.725477:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306340281, at 4306340288
00000100:00000010:0.0:1788574736.725491:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00000040:1.0:1788574736.725532:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306340281, at 4306340288
00000100:00000010:1.0:1788574736.725548:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7a80.
00000100:00000001:0.0:1788574737.301492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574737.301499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574737.301502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574737.301508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574737.301512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574737.301516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574737.301519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574737.301521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574737.301525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574738.325539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574738.325547:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574738.325549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574738.325555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574738.325559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574738.325563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574738.325566:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574738.325569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574738.325572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574739.349494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574739.349501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574739.349502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574739.349506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574739.349509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574739.349512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574739.349514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574739.349516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574739.349518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.373523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574740.373529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574740.373534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574740.373538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574740.373541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574740.373545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.373547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574740.373549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.373551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.885714:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574740.885720:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574740.885724:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574740.885733:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.885749:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574740.885752:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574740.885758:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574740.885761:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574740.885768:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574740.885773:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393157120:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574740.885778:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574740.885784:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574740.885789:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574740.885795:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574740.885803:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574740.885810:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574740.885827:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574740.885832:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574740.885841:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574740.885854:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393157120, portal 25
00000100:00000001:0.0:1788574740.885861:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574740.885865:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000100:00000040:0.0:1788574740.885879:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.885904:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574740.885910:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574740.885917:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574740.885923:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393157120, offset 0
00000400:00000200:0.0:1788574740.885934:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574740.885945:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574740.885956:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574740.885964:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574740.885973:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574740.885980:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574740.885985:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574740.885991:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330483:330483:256:4294967295] 192.168.202.110@tcp LPNI seq info [330440:330440:8:4294967295]
00000400:00000200:0.0:1788574740.886001:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574740.886016:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574740.886027:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574740.886034:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574740.886045:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574740.886050:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788574740.886062:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574740.886068:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1788574740.886076:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574740.886080:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574740.886088:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574740.886091:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574740.886096:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574740.886099:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574740.886103:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574740.886107:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886115:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574740.886118:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886123:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.886124:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574740.886128:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574740.886131:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574740.886133:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306345401
00000100:00000010:0.0:1788574740.886136:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d77e0.
00000100:00000001:0.0:1788574740.886142:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788574740.886147:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306345401
00000100:00000010:0.0:1788574740.886149:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d78a0.
00000100:00000001:0.0:1788574740.886151:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788574740.886154:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574740.886156:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.886158:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574740.886161:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574740/real 1788574740]  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574740.886168:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574740.886169:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574740.886171:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574740.886173:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574740.886174:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574740.886175:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574740.886177:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574740.886183:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886188:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574740.886190:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886191:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574740.886193:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574740.886196:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886200:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574740.886203:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886207:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574740.886209:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886210:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574740.886211:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886213:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574740.886214:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574740.886217:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574740.886219:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574740.886221:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574740.886224:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574740.886229:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393157120:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788574740.886232:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886259:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574740.886264:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393157120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886268:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574740.886270:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574740.886273:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574740.886275:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574740.886276:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574740.886277:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788574740.886279:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574740.886281:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788574740.886283:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574740.886285:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574740.886287:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:0.0:1788574740.886289:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574740.886290:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886291:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886297:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574740.886299:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.886300:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574740.886301:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574740.886303:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574740.886305:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886310:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574740.886311:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574740.886312:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574740.886314:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574740.886315:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574740.886317:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393157248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574740.886320:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574740.886321:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574740.886322:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574740.886324:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574740.886326:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574740.886327:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574740.886330:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574740.886332:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574740.886334:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574740.886337:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393157248, portal 10
00000100:00000001:0.0:1788574740.886338:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574740.886339:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:0.0:1788574740.886344:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886348:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574740.886350:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574740.886352:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574740.886353:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393157248, offset 0
00000400:00000200:0.0:1788574740.886356:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574740.886359:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574740.886361:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574740.886363:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574740.886365:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574740.886367:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574740.886369:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574740.886371:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330484:330484:256:4294967295] 192.168.202.110@tcp LPNI seq info [330441:330441:8:4294967295]
00000400:00000200:0.0:1788574740.886374:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574740.886378:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574740.886381:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574740.886383:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574740.886386:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574740.886388:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788574740.886416:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574740.886418:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1788574740.886421:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574740.886422:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574740.886424:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574740.886427:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574740.886430:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574740.886432:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574740.886434:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574740.886438:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886452:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574740.886456:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886463:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.886465:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574740.886468:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574740.886472:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788574740.886476:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574740.886478:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306344487
00000100:00000001:0.0:1788574740.886481:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306345401
00000100:00000001:0.0:1788574740.886484:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306344487
00000100:00000001:0.0:1788574740.886485:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306345401
00000100:00000001:0.0:1788574740.886487:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574740.886488:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.886490:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574740.886493:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574740/real 1788574740]  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574740.886499:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574740.886500:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574740.886501:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574740.886503:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574740.886504:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574740.886506:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574740.886508:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574740.886513:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886519:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574740.886521:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886523:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574740.886524:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574740.886528:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886535:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574740.886540:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886547:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574740.886550:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886552:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574740.886553:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886555:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574740.886557:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574740.886560:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574740.886563:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574740.886566:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574740.886570:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574740.886575:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393157248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788574740.886578:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788574740.886581:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886586:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574740.886586:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574740.886588:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788574740.886589:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574740.886591:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574740.886593:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393157376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574740.886595:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788574740.886597:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788574740.886598:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574740.886599:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788574740.886602:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574740.886604:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788574740.886606:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788574740.886608:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788574740.886618:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788574740.886621:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393157376, portal 4
00000100:00000001:0.0:1788574740.886622:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788574740.886623:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:0.0:1788574740.886628:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886632:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788574740.886634:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788574740.886635:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788574740.886637:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393157376, offset 0
00000400:00000200:0.0:1788574740.886639:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788574740.886642:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574740.886644:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788574740.886647:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788574740.886648:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788574740.886650:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788574740.886652:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788574740.886654:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330485:330485:256:4294967295] 192.168.202.110@tcp LPNI seq info [330442:330442:8:4294967295]
00000400:00000200:0.0:1788574740.886657:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574740.886662:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788574740.886665:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574740.886667:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788574740.886670:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788574740.886672:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1788574740.886675:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788574740.886677:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1788574740.886679:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788574740.886681:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788574740.886683:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788574740.886685:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788574740.886688:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788574740.886689:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788574740.886690:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788574740.886692:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886698:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574740.886700:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886704:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.886705:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574740.886707:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574740.886709:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788574740.886711:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788574740.886713:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306344633
00000100:00000001:0.0:1788574740.886715:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306345401
00000100:00000001:0.0:1788574740.886716:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306344633
00000100:00000001:0.0:1788574740.886718:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306345401
00000100:00000001:0.0:1788574740.886719:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788574740.886720:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.886721:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788574740.886724:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574740/real 1788574740]  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788574740.886729:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788574740.886730:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788574740.886731:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788574740.886733:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788574740.886734:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788574740.886735:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788574740.886737:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788574740.886742:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886746:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788574740.886747:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886748:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574740.886749:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788574740.886751:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886755:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788574740.886757:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886761:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788574740.886763:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886764:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788574740.886765:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886766:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788574740.886767:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788574740.886769:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788574740.886771:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788574740.886773:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788574740.886775:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788574740.886784:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393157376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788574740.886786:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886787:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574740.886789:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393157248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886793:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574740.886794:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574740.886796:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788574740.886797:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574740.886798:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574740.886799:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788574740.886800:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574740.886801:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788574740.886803:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574740.886804:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574740.886805:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:0.0:1788574740.886807:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574740.886808:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886809:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788574740.886811:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393157376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574740.886814:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788574740.886815:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788574740.886816:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788574740.886817:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788574740.886818:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788574740.886819:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788574740.886820:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788574740.886821:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:0.0:1788574740.886822:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788574740.886822:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788574740.886823:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788574740.886826:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788574740.886826:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886828:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574740.886830:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574740.886831:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574740.886832:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788574740.886834:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574741.845509:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306345401, at 4306345408
00000100:00000040:1.0:1788574741.845509:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306345401, at 4306345408
00000100:00000010:1.0:1788574741.845533:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d78a0.
00000100:00000010:0.0:1788574741.845536:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d77e0.
00000100:00080000:0.0:1788574746.005502:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574746.005511:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574746.005513:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574746.005516:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574746.005520:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574746.005525:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11677
00000400:00000200:0.0:1788574746.005533:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574746.005538:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574746.005540:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574746.005542:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005545:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574746.005547:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005549:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574746.005552:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574746.005553:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574746.005555:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574746.005557:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574746.005559:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574746.005560:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574746.005562:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005563:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574746.005565:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005566:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574746.005569:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574746.005572:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005574:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574746.005576:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574746.005578:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574746.005580:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574746.005582:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574746.005583:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574746.005591:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574746.005593:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574746.005595:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574746.005596:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574746.005599:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574746.005601:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574746.005604:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574746.005610:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574746.005615:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574746.005630:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393157504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574746.005637:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574746.005641:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574746.005644:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393157504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574746.005661:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574746.005664:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574746.005666:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574746.005667:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574746.005669:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574746.005670:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574746.005673:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11677
00000400:00000200:0.0:1788574746.005675:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574746.005677:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574746.005679:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574746.005681:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005682:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574746.005684:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005685:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574746.005686:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574746.005687:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574746.005688:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574746.005689:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574746.005690:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574746.005692:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574746.005694:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005695:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574746.005697:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005698:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574746.005701:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574746.005702:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005704:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574746.005705:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574746.005709:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574746.005711:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574746.005712:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574746.005716:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574746.005718:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574746.005719:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574746.005721:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788574746.005723:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574746.005725:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574746.005726:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574746.005728:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574746.005729:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574746.005731:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574746.005734:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574746.005736:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574746.005741:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393157632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574746.005746:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574746.005752:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574746.005756:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393157632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574746.005766:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574746.005769:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574746.005770:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574746.005785:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574746.005786:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574746.005788:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574746.005792:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11677
00000400:00000200:0.0:1788574746.005797:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574746.005800:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574746.005807:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574746.005814:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005822:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574746.005827:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005836:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574746.005845:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574746.005854:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574746.005857:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574746.005861:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574746.005863:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574746.005867:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574746.005874:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005877:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574746.005882:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005885:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574746.005894:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574746.005905:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574746.005916:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574746.005923:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574746.005927:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574746.005932:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574746.005933:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574746.005935:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574746.005936:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574746.005938:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574746.005940:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788574746.005943:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574746.005948:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574746.005950:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574746.005958:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574746.005959:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574746.005962:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574746.005967:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788574746.005969:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574746.005975:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393157760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574746.005990:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574746.005993:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574746.005996:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393157760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574746.006015:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574746.006018:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574746.006022:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574746.006033:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11687)
00000800:00000200:2.1:1788574746.006205:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574746.006210:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574746.006711:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574746.006714:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574746.007031:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574746.007033:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:0.0:1788574746.965471:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306350521, at 4306350528
00000100:00000040:1.0:1788574746.965471:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306350521, at 4306350528
00000100:00000010:1.0:1788574746.965488:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000100:00000010:0.0:1788574746.965493:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000100:00000001:0.0:1788574748.565488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574748.565494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574748.565496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574748.565501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574748.565504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574748.565507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574748.565510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574748.565512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574748.565515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574749.589559:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574749.589566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574749.589568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574749.589573:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574749.589577:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574749.589581:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574749.589584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574749.589587:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574749.589589:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574750.613555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574750.613566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574750.613570:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574750.613577:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574750.613581:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574750.613585:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574750.613587:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574750.613590:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574750.613593:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574751.125529:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574751.125536:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574751.125541:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574751.125725:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574751.125749:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:0.0:1788574751.125753:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574751.125755:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574751.125760:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:3.1:1788574751.126192:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574751.126196:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788574751.126564:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574751.126565:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788574751.126791:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574751.126794:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1788574751.637574:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574751.637582:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574751.637584:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574751.637589:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574751.637593:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574751.637597:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574751.637599:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574751.637601:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574751.637604:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574752.085558:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306355641, at 4306355648
00000100:00000040:1.0:1788574752.085567:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306355641, at 4306355648
00000100:00000010:0.0:1788574752.085578:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:1.0:1788574752.085585:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:0.0:1788574752.661561:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574752.661570:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574752.661573:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574752.661577:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574752.661583:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574752.661587:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574752.661590:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574752.661593:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574752.661596:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574753.685523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574753.685530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574753.685532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574753.685536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574753.685539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574753.685542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574753.685544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574753.685547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574753.685548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574754.709520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574754.709526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574754.709527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574754.709532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574754.709534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574754.709537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574754.709539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574754.709541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574754.709544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574755.733481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574755.733488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574755.733490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574755.733495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574755.733499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574755.733503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574755.733506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574755.733508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574755.733511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574756.246066:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574756.246072:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788574756.246483:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574756.246484:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788574756.246704:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574756.246705:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1788574756.757469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574756.757476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574756.757478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574756.757482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574756.757485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574756.757488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574756.757490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574756.757492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574756.757494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574757.205476:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306360761, at 4306360768
00000100:00000040:1.0:1788574757.205476:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306360761, at 4306360768
00000100:00000010:1.0:1788574757.205492:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788574757.205497:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000001:0.0:1788574757.781515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574757.781523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574757.781525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574757.781531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574757.781535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574757.781539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574757.781542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574757.781545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574757.781548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574758.805476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574758.805483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574758.805484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574758.805488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574758.805491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574758.805494:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574758.805496:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574758.805498:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574758.805501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574759.829465:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574759.829471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574759.829473:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574759.829478:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574759.829480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574759.829483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574759.829485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574759.829487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574759.829490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574760.853525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574760.853532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574760.853533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574760.853538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574760.853541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574760.853546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574760.853549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574760.853552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574760.853555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574761.365989:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574761.365992:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788574761.366310:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574761.366312:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788574761.366597:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574761.366600:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:0.0:1788574761.877538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574761.877545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574761.877547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574761.877551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574761.877554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574761.877565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574761.877567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574761.877569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574761.877572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574762.325503:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306365881, at 4306365888
00000100:00000040:1.0:1788574762.325503:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306365881, at 4306365888
00000100:00000010:0.0:1788574762.325530:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:1.0:1788574762.325532:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000001:0.0:1788574762.901497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574762.901506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574762.901509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574762.901516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574762.901521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574762.901524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574762.901527:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574762.901530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574762.901533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574763.925514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574763.925521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574763.925523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574763.925529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574763.925532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574763.925535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574763.925537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574763.925539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574763.925541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574764.949526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574764.949542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574764.949545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574764.949552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574764.949557:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574764.949562:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574764.949565:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574764.949568:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574764.949572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574765.973512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574765.973520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574765.973523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574765.973529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574765.973532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574765.973535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574765.973537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574765.973539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574765.973542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574766.486322:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574766.486331:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788574766.486885:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574766.486888:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788574766.487178:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574766.487181:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1788574766.997555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574766.997561:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574766.997563:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574766.997568:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574766.997572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574766.997576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574766.997580:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574766.997583:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574766.997586:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574767.445516:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306371002, at 4306371008
00000100:00000040:1.0:1788574767.445516:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306371002, at 4306371008
00000100:00000010:1.0:1788574767.445531:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788574767.445534:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000001:0.0:1788574768.021488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574768.021497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574768.021500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574768.021507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574768.021511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574768.021515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574768.021518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574768.021521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574768.021523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574769.045521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574769.045527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574769.045529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574769.045533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574769.045536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574769.045539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574769.045541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574769.045543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574769.045545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574770.069528:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574770.069536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574770.069539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574770.069547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574770.069551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574770.069555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574770.069558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574770.069561:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574770.069564:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574771.093500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574771.093507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574771.093509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574771.093521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574771.093524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574771.093527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574771.093529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574771.093531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574771.093533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574771.605507:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574771.605515:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574771.605518:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574771.605521:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574771.605525:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574771.605531:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11702
00000400:00000200:0.0:1788574771.605538:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574771.605543:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574771.605545:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574771.605548:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605550:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574771.605552:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605555:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574771.605559:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574771.605561:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574771.605564:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574771.605567:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574771.605570:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574771.605572:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574771.605576:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605578:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574771.605580:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605582:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574771.605586:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574771.605589:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605592:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574771.605595:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574771.605599:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574771.605602:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574771.605604:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574771.605607:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574771.605615:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574771.605617:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574771.605621:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574771.605623:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574771.605625:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574771.605628:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574771.605631:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574771.605637:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788574771.605641:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574771.605649:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393159424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574771.605656:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574771.605660:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574771.605663:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393159424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574771.605682:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574771.605685:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574771.605687:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574771.605689:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574771.605690:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574771.605692:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574771.605694:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11702
00000400:00000200:0.0:1788574771.605697:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574771.605699:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574771.605701:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574771.605704:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605705:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574771.605707:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605709:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574771.605711:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574771.605712:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574771.605713:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574771.605715:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574771.605717:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574771.605721:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574771.605725:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605726:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574771.605728:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605729:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574771.605731:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574771.605732:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605734:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574771.605735:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574771.605740:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574771.605744:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574771.605745:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574771.605749:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574771.605752:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574771.605755:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574771.605759:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574771.605762:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574771.605767:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574771.605768:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574771.605772:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574771.605774:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574771.605776:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574771.605781:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788574771.605787:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574771.605793:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393159552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574771.605804:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574771.605807:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574771.605838:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393159552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574771.605857:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574771.605870:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574771.605876:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574771.605880:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574771.605884:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574771.605890:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574771.605896:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11702
00000400:00000200:0.0:1788574771.605904:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574771.605908:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574771.605911:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574771.605914:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605919:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574771.605922:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605925:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574771.605930:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574771.605932:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574771.605933:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574771.605936:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574771.605938:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574771.605947:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574771.605953:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605956:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574771.605961:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605964:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574771.605971:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574771.605973:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574771.605979:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574771.605981:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574771.605984:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574771.605986:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574771.605990:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574771.605993:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574771.606005:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574771.606009:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574771.606016:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788574771.606021:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574771.606027:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574771.606031:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574771.606037:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574771.606040:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574771.606043:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574771.606048:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788574771.606052:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574771.606061:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393159680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574771.606074:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574771.606077:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574771.606084:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393159680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574771.606101:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574771.606106:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574771.606111:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574771.606116:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11712)
00000800:00000200:3.1:1788574771.606147:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574771.606151:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788574771.606686:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574771.606689:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788574771.607077:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574771.607080:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:0.0:1788574772.117471:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574772.117483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574772.117485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574772.117491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574772.117495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574772.117499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574772.117502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574772.117505:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574772.117508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574772.565516:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306376121, at 4306376128
00000100:00000010:1.0:1788574772.565530:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000040:0.0:1788574772.565629:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306376121, at 4306376128
00000100:00000010:0.0:1788574772.565636:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:0.0:1788574773.141487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574773.141494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574773.141495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574773.141500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574773.141502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574773.141505:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574773.141507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574773.141509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574773.141511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574774.165522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574774.165530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574774.165533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574774.165539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574774.165543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574774.165547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574774.165550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574774.165552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574774.165555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574775.189542:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574775.189550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574775.189552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574775.189557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574775.189560:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574775.189565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574775.189567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574775.189569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574775.189572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574776.213533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574776.213542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574776.213544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574776.213550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574776.213554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574776.213558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574776.213561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574776.213563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574776.213566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574776.726169:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574776.726417:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574776.726553:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574777.237534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574777.237540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574777.237542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574777.237546:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574777.237550:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574777.237554:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574777.237556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574777.237563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574777.237565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574777.685534:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306381241, at 4306381248
00000100:00000040:1.0:1788574777.685543:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306381241, at 4306381248
00000100:00000010:0.0:1788574777.685551:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:1.0:1788574777.685558:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:0.0:1788574778.261563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574778.261570:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574778.261573:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574778.261579:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574778.261583:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574778.261586:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574778.261588:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574778.261591:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574778.261594:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574779.285532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574779.285539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574779.285541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574779.285545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574779.285549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574779.285552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574779.285555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574779.285557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574779.285560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574780.309486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574780.309494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574780.309496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574780.309506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574780.309510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574780.309514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574780.309516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574780.309519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574780.309521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574781.333479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574781.333487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574781.333489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574781.333494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574781.333497:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574781.333501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574781.333503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574781.333506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574781.333509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574781.846075:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574781.846081:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788574781.846618:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574781.846621:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788574781.846905:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574781.846908:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1788574782.357499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574782.357506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574782.357509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574782.357514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574782.357518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574782.357522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574782.357525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574782.357527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574782.357530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574782.805440:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306386361, at 4306386368
00000100:00000040:0.0:1788574782.805447:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306386361, at 4306386368
00000100:00000010:1.0:1788574782.805453:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000010:0.0:1788574782.805455:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000001:0.0:1788574783.381518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574783.381523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574783.381525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574783.381529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574783.381531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574783.381535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574783.381536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574783.381539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574783.381541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574784.405492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574784.405500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574784.405503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574784.405508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574784.405511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574784.405514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574784.405517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574784.405519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574784.405522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574785.429525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574785.429531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574785.429534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574785.429539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574785.429542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574785.429545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574785.429547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574785.429550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574785.429553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574786.453522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574786.453530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574786.453533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574786.453538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574786.453542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574786.453545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574786.453548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574786.453550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574786.453553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574786.965539:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574786.965551:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574786.965554:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574786.965560:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574786.965566:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574786.965575:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11718
00000400:00000200:1.0:1788574786.965586:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574786.965593:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574786.965596:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574786.965599:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965603:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574786.965606:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965608:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574786.965613:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574786.965615:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574786.965618:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574786.965621:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574786.965624:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574786.965626:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574786.965628:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965630:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574786.965633:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965635:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574786.965639:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574786.965642:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965645:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574786.965648:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574786.965651:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574786.965655:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574786.965657:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574786.965660:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574786.965672:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788574786.965676:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574786.965680:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574786.965681:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574786.965684:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574786.965687:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574786.965692:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574786.965701:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788574786.965709:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574786.965718:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393160576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574786.965729:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574786.965734:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574786.965738:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393160576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574786.965760:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574786.965764:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574786.965767:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574786.965769:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574786.965771:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574786.965773:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574786.965775:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11718
00000400:00000200:1.0:1788574786.965779:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574786.965782:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574786.965785:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574786.965788:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965791:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574786.965793:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965795:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574786.965800:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574786.965802:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574786.965805:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574786.965807:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574786.965810:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574786.965813:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574786.965819:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965823:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574786.965827:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965831:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574786.965836:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574786.965847:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574786.965850:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574786.965853:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574786.965861:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574786.965864:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574786.965866:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574786.965873:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574786.965877:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574786.965879:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574786.965884:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788574786.965888:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574786.965892:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574786.965893:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574786.965898:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574786.965905:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574786.965908:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574786.965913:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788574786.965917:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574786.965923:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393160704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574786.965936:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574786.965939:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574786.965946:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393160704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574786.965964:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574786.965969:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574786.965974:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574786.965977:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574786.965980:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574786.965983:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574786.965988:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11718
00000400:00000200:1.0:1788574786.965994:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574786.965997:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574786.965999:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574786.966005:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.966008:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574786.966011:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.966014:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574786.966019:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574786.966021:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574786.966023:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574786.966025:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574786.966030:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574786.966032:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574786.966038:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.966041:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574786.966045:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574786.966050:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574786.966054:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574786.966059:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574786.966064:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574786.966067:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574786.966070:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574786.966075:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574786.966077:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574786.966079:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574786.966082:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574786.966086:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574786.966092:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788574786.966096:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574786.966100:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574786.966103:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574786.966107:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574786.966110:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574786.966111:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574786.966116:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574786.966120:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574786.966127:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393160832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574786.966138:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574786.966142:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574786.966143:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788574786.966148:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393160832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1788574786.966149:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788574786.966162:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574786.966166:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574786.966172:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574786.966177:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11728)
00000800:00000200:3.1:1788574786.966664:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574786.966666:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788574786.966964:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574786.966967:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:0.0:1788574787.477767:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574787.477779:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574787.477782:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574787.477790:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574787.477795:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574787.477799:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574787.477802:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574787.477805:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574787.477809:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574787.925474:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306391481, at 4306391488
00000100:00000010:1.0:1788574787.925486:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000040:0.0:1788574787.925507:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306391481, at 4306391488
00000100:00000010:0.0:1788574787.925522:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000001:0.0:1788574788.501510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574788.501516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574788.501517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574788.501521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574788.501523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574788.501527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574788.501529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574788.501530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574788.501532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574789.525554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574789.525569:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574789.525572:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574789.525578:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574789.525582:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574789.525585:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574789.525588:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574789.525591:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574789.525593:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000001:1.0:1788574790.037674:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574790.037678:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574790.037687:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574790.037692:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574790.037695:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574790.549505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574790.549510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574790.549512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574790.549515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574790.549517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574790.549520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574790.549521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574790.549523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574790.549524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574791.573550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574791.573557:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574791.573559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574791.573564:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574791.573570:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574791.573573:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574791.573576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574791.573579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574791.573581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574792.086065:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574792.086072:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788574792.086451:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574792.086453:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788574792.086630:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574792.086631:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1788574792.597537:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574792.597543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574792.597546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574792.597550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574792.597554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574792.597558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574792.597561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574792.597563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574792.597566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574793.045516:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306396601, at 4306396608
00000100:00000010:1.0:1788574793.045529:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00000100:00000040:0.0:1788574793.045612:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306396601, at 4306396608
00000100:00000010:0.0:1788574793.045617:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000100:00000001:0.0:1788574793.621526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574793.621533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574793.621535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574793.621539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574793.621541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574793.621544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574793.621547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574793.621549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574793.621551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574794.645549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574794.645556:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574794.645558:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574794.645563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574794.645568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574794.645571:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574794.645574:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574794.645578:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574794.645581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574795.669507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574795.669514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574795.669517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574795.669521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574795.669525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574795.669529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574795.669531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574795.669534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574795.669538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574796.693520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574796.693526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574796.693528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574796.693533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574796.693536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574796.693539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574796.693541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574796.693543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574796.693545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574797.206015:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574797.206019:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788574797.206529:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574797.206531:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788574797.206812:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574797.206814:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:0.0:1788574797.717501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574797.717507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574797.717509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574797.717513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574797.717516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574797.717519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574797.717521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574797.717523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574797.717525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574798.165487:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306401721, at 4306401728
00000100:00000040:1.0:1788574798.165487:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306401721, at 4306401728
00000100:00000010:1.0:1788574798.165504:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:0.0:1788574798.165511:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000001:0.0:1788574798.741519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574798.741527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574798.741529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574798.741536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574798.741539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574798.741544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574798.741547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574798.741550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574798.741553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574799.765497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574799.765502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574799.765504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574799.765508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574799.765510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574799.765513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574799.765522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574799.765524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574799.765529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574800.789556:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574800.789563:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574800.789565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574800.789571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574800.789574:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574800.789577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574800.789580:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574800.789582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574800.789584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574801.813488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574801.813496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574801.813499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574801.813504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574801.813506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574801.813509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574801.813713:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574801.813717:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574801.813721:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574802.325541:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574802.325553:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574802.325556:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574802.325562:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574802.325567:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574802.325579:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11733
00000400:00000200:0.0:1788574802.325590:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574802.325597:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574802.325601:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574802.325604:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325608:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574802.325610:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325613:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574802.325617:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574802.325619:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574802.325622:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574802.325625:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574802.325627:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574802.325629:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574802.325631:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325634:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574802.325635:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325637:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574802.325639:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574802.325642:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325644:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574802.325647:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574802.325650:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574802.325653:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574802.325655:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574802.325658:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574802.325671:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788574802.325676:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574802.325681:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574802.325683:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574802.325687:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574802.325690:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574802.325696:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574802.325707:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788574802.325717:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574802.325728:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393161728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574802.325737:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574802.325741:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574802.325745:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393161728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574802.325766:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574802.325768:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574802.325771:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574802.325773:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574802.325774:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574802.325777:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574802.325781:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11733
00000400:00000200:0.0:1788574802.325785:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574802.325789:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574802.325793:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574802.325796:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325801:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574802.325803:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325807:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574802.325809:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574802.325811:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574802.325812:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574802.325815:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574802.325817:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574802.325821:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574802.325827:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325831:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574802.325835:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325839:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574802.325842:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574802.325847:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574802.325852:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574802.325857:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574802.325863:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574802.325868:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574802.325886:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574802.325891:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574802.325895:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574802.325896:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574802.325901:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788574802.325905:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574802.325910:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574802.325913:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574802.325915:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574802.325917:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574802.325919:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574802.325923:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788574802.325928:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574802.325938:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393161856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574802.325957:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574802.325964:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574802.325971:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393161856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574802.325987:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574802.325992:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574802.325997:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574802.325998:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574802.326000:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574802.326012:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574802.326020:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11733
00000400:00000200:0.0:1788574802.326029:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574802.326036:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574802.326042:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574802.326051:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.326056:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574802.326063:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.326069:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574802.326077:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574802.326080:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574802.326084:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574802.326089:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574802.326091:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574802.326095:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574802.326101:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.326106:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574802.326109:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574802.326112:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574802.326116:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574802.326124:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574802.326128:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574802.326134:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574802.326140:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574802.326145:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574802.326150:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574802.326155:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574802.326163:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574802.326166:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574802.326175:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788574802.326181:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574802.326189:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574802.326192:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574802.326195:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574802.326199:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574802.326201:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574802.326206:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574802.326210:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574802.326219:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393161984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1788574802.326234:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788574802.326235:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1788574802.326238:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:0.0:1788574802.326241:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574802.326251:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393161984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574802.326269:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574802.326275:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574802.326281:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574802.326289:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11743)
00000800:00000200:3.1:1788574802.326658:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574802.326661:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788574802.326935:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574802.326937:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:0.0:1788574802.837607:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574802.837616:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574802.837618:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574802.837624:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574802.837628:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574802.837631:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574802.837634:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574802.837637:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574802.837640:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574803.285528:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306406841, at 4306406848
00000100:00000040:1.0:1788574803.285528:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306406841, at 4306406848
00000100:00000010:1.0:1788574803.285546:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:0.0:1788574803.285549:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000001:0.0:1788574803.861478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574803.861486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574803.861489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574803.861494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574803.861498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574803.861502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574803.861505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574803.861507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574803.861510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574804.885515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574804.885522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574804.885524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574804.885530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574804.885533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574804.885537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574804.885540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574804.885544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574804.885547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574805.909506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574805.909513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574805.909516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574805.909522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574805.909526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574805.909530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574805.909533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574805.909537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574805.909539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574806.933555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574806.933563:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574806.933565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574806.933570:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574806.933572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574806.933575:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574806.933577:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574806.933579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574806.933581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574807.445539:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574807.445548:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574807.445551:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574807.445554:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574807.445558:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574807.445563:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11738
00000400:00000200:1.0:1788574807.445571:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574807.445577:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574807.445579:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574807.445582:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445584:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574807.445586:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445588:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574807.445591:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574807.445593:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574807.445595:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574807.445597:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574807.445599:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574807.445600:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574807.445602:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445603:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574807.445604:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445606:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574807.445608:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574807.445611:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445613:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574807.445615:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574807.445618:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574807.445620:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574807.445622:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574807.445624:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574807.445632:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788574807.445635:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574807.445638:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574807.445639:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574807.445642:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574807.445644:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574807.445648:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574807.445654:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574807.445660:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574807.445667:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393162112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574807.445675:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574807.445679:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574807.445682:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393162112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574807.445700:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574807.445702:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574807.445705:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574807.445706:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574807.445707:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574807.445709:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574807.445711:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11738
00000400:00000200:1.0:1788574807.445713:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574807.445715:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574807.445734:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574807.445737:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445740:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574807.445742:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445744:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574807.445747:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574807.445748:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574807.445750:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574807.445752:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574807.445754:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574807.445755:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574807.445758:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445760:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574807.445762:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445766:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574807.445768:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574807.445773:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445777:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574807.445778:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574807.445784:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574807.445786:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574807.445788:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574807.445791:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574807.445795:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574807.445799:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574807.445804:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788574807.445809:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574807.445814:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574807.445818:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574807.445823:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574807.445826:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574807.445831:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574807.445839:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788574807.445848:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574807.445859:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393162240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574807.445870:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574807.445877:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574807.445887:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393162240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574807.445902:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574807.445906:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574807.445913:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574807.445915:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574807.445919:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574807.445925:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574807.445930:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11738
00000400:00000200:1.0:1788574807.445936:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574807.445942:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574807.445946:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574807.445952:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445956:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574807.445959:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445963:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574807.445966:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574807.445969:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574807.445971:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574807.445974:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574807.445977:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574807.445980:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574807.445984:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445987:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574807.445991:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574807.445994:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574807.445996:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574807.445998:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574807.446000:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574807.446003:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574807.446008:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574807.446010:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574807.446011:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574807.446014:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574807.446017:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574807.446021:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574807.446025:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788574807.446030:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574807.446036:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574807.446039:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574807.446045:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574807.446050:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574807.446054:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574807.446062:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574807.446068:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574807.446079:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393162368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574807.446090:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574807.446094:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574807.446098:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393162368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574807.446109:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574807.446112:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574807.446117:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574807.446121:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11748)
00000800:00000200:3.1:1788574807.446137:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574807.446142:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1788574807.446609:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574807.446612:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1788574807.446974:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574807.446976:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:0.0:1788574807.957529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574807.957536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574807.957543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574807.957549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574807.957553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574807.957557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574807.957559:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574807.957562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574807.957566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574808.405504:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306411961, at 4306411968
00000100:00000040:1.0:1788574808.405504:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306411961, at 4306411968
00000100:00000010:1.0:1788574808.405523:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788574808.405528:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000001:0.0:1788574808.981505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574808.981512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574808.981514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574808.981520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574808.981523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574808.981527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574808.981530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574808.981532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574808.981535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574810.005534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574810.005543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574810.005545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574810.005551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574810.005555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574810.005559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574810.005561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574810.005563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574810.005566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574811.029491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574811.029500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574811.029502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574811.029507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574811.029510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574811.029514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574811.029516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574811.029519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574811.029522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574812.053498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574812.053505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574812.053508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574812.053513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574812.053516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574812.053520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574812.053522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574812.053525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574812.053527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574812.565466:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574812.565474:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574812.565476:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574812.565485:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574812.565488:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574812.565494:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11743
00000400:00000200:1.0:1788574812.565502:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574812.565506:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574812.565508:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574812.565510:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565513:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574812.565514:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565517:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574812.565521:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574812.565523:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574812.565525:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574812.565528:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574812.565531:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574812.565533:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574812.565535:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565537:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574812.565539:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565541:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574812.565544:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574812.565547:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565549:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574812.565552:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574812.565555:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574812.565558:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574812.565571:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574812.565573:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574812.565581:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788574812.565585:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574812.565589:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574812.565590:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574812.565593:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574812.565596:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574812.565600:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574812.565607:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574812.565612:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574812.565620:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393162496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574812.565626:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574812.565629:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574812.565632:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393162496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574812.565649:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574812.565651:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574812.565653:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574812.565654:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574812.565655:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574812.565657:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574812.565659:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11743
00000400:00000200:1.0:1788574812.565662:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574812.565664:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574812.565665:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574812.565667:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565668:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574812.565670:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565671:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574812.565673:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574812.565673:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574812.565674:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574812.565676:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574812.565678:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574812.565679:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574812.565681:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565683:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574812.565684:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565685:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574812.565686:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574812.565688:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565690:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574812.565691:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574812.565696:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574812.565698:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574812.565701:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574812.565706:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574812.565708:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574812.565709:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574812.565719:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788574812.565722:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574812.565726:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574812.565728:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574812.565729:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574812.565732:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574812.565733:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574812.565739:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574812.565741:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574812.565747:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393162624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574812.565758:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574812.565760:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574812.565767:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393162624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574812.565780:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574812.565782:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574812.565785:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574812.565786:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574812.565787:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574812.565789:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574812.565793:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11743
00000400:00000200:1.0:1788574812.565798:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574812.565800:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574812.565804:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574812.565809:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565810:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574812.565813:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565816:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574812.565817:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574812.565818:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574812.565819:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574812.565821:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574812.565826:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574812.565827:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574812.565829:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565830:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574812.565832:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565836:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574812.565840:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574812.565845:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574812.565847:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574812.565850:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574812.565853:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574812.565856:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574812.565860:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574812.565862:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574812.565866:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574812.565874:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574812.565879:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788574812.565882:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574812.565888:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574812.565889:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574812.565890:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574812.565893:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574812.565896:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574812.565899:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788574812.565903:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574812.565909:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393162752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574812.565919:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574812.565921:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574812.565924:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393162752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574812.565935:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574812.565938:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574812.565943:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574812.565946:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11753)
00000800:00000200:3.1:1788574812.565999:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574812.566002:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1788574812.566364:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574812.566366:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1788574812.566667:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574812.566670:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00000001:0.0:1788574813.077567:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574813.077572:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574813.077573:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574813.077577:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574813.077581:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574813.077583:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574813.077585:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574813.077587:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574813.077590:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574813.525492:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306417081, at 4306417088
00000100:00000040:1.0:1788574813.525492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306417081, at 4306417088
00000100:00000010:0.0:1788574813.525509:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:1.0:1788574813.525511:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000001:0.0:1788574814.101503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574814.101510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574814.101512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574814.101532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574814.101535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574814.101539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574814.101542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574814.101544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574814.101547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574815.125507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574815.125515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574815.125517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574815.125521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574815.125525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574815.125528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574815.125531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574815.125533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574815.125536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574816.149539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574816.149545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574816.149548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574816.149552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574816.149555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574816.149559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574816.149561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574816.149564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574816.149566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574817.173494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574817.173501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574817.173503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574817.173508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574817.173512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574817.173516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574817.173519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574817.173521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574817.173524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574817.685504:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574817.685514:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574817.685517:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574817.685521:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574817.685526:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574817.685533:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11748
00000400:00000200:1.0:1788574817.685543:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574817.685548:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574817.685551:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574817.685555:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685559:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574817.685561:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685563:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574817.685566:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574817.685568:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574817.685570:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574817.685573:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574817.685575:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574817.685577:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574817.685580:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685582:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574817.685583:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685585:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574817.685587:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574817.685590:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685592:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574817.685594:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574817.685597:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574817.685599:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574817.685601:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574817.685603:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574817.685611:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788574817.685614:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574817.685617:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574817.685618:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574817.685621:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574817.685623:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574817.685627:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574817.685634:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574817.685640:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574817.685648:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393162880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574817.685658:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574817.685663:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574817.685667:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393162880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574817.685687:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574817.685690:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574817.685693:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574817.685694:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574817.685696:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574817.685698:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574817.685700:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11748
00000400:00000200:1.0:1788574817.685703:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574817.685705:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574817.685708:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574817.685711:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685713:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574817.685716:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685717:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574817.685720:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574817.685721:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574817.685722:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574817.685724:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574817.685727:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574817.685732:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574817.685737:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685741:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574817.685745:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685747:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574817.685749:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574817.685756:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685762:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574817.685767:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574817.685773:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574817.685778:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574817.685781:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574817.685786:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574817.685792:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574817.685796:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574817.685801:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788574817.685805:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574817.685810:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574817.685813:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574817.685816:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574817.685818:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574817.685820:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574817.685825:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574817.685829:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574817.685836:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393163008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574817.685849:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574817.685854:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574817.685860:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393163008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574817.685883:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574817.685888:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574817.685894:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574817.685898:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574817.685902:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574817.685909:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574817.685924:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11748
00000400:00000200:1.0:1788574817.685931:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574817.685939:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574817.685944:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574817.685954:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685958:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574817.685964:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685967:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574817.685971:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574817.685973:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574817.685978:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574817.685981:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574817.685985:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574817.685990:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574817.685996:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.685999:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574817.686003:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574817.686004:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574817.686007:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574817.686010:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574817.686013:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574817.686016:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574817.686020:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574817.686025:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574817.686028:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.1:1788574817.686032:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574817.686035:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000008:00100000:1.0:1788574817.686037:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574817.686044:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574817.686048:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574817.686053:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788574817.686057:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574817.686063:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574817.686066:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574817.686068:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574817.686072:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574817.686076:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574817.686083:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574817.686089:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574817.686099:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393163136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574817.686112:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574817.686116:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574817.686125:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393163136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574817.686137:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574817.686143:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574817.686150:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574817.686154:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11758)
00000800:00000200:3.1:1788574817.686418:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574817.686420:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1788574817.686667:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574817.686669:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000100:00000001:0.0:1788574818.197500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574818.197505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574818.197506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574818.197511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574818.197513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574818.197516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574818.197518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574818.197520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574818.197522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574818.645516:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306422201, at 4306422208
00000100:00000040:0.0:1788574818.645524:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306422201, at 4306422208
00000100:00000010:1.0:1788574818.645532:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:0.0:1788574818.645536:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:0.0:1788574819.221506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574819.221513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574819.221515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574819.221520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574819.221524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574819.221528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574819.221530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574819.221532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574819.221534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574820.245504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574820.245509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574820.245511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574820.245514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574820.245518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574820.245521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574820.245524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574820.245526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574820.245528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574821.269451:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574821.269458:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574821.269461:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574821.269466:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574821.269468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574821.269471:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574821.269473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574821.269474:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574821.269476:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574822.293490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574822.293496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574822.293498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574822.293504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574822.293507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574822.293510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574822.293512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574822.293514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574822.293517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574822.805515:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574822.805525:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574822.805528:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574822.805532:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574822.805537:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574822.805543:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11753
00000400:00000200:0.0:1788574822.805551:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574822.805557:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574822.805560:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574822.805563:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805566:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574822.805568:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805570:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574822.805574:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574822.805575:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574822.805578:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574822.805581:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574822.805583:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574822.805586:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574822.805588:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805590:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574822.805592:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805594:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574822.805597:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574822.805600:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805603:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574822.805606:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574822.805609:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574822.805611:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574822.805613:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574822.805615:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574822.805624:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574822.805627:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574822.805631:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574822.805632:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574822.805635:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574822.805637:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574822.805641:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574822.805648:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574822.805654:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574822.805663:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393163264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574822.805672:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574822.805675:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574822.805679:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393163264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574822.805700:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574822.805703:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574822.805707:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574822.805708:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574822.805710:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574822.805712:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574822.805715:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11753
00000400:00000200:0.0:1788574822.805719:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574822.805721:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574822.805724:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574822.805727:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805730:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574822.805732:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805734:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574822.805737:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574822.805741:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574822.805744:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574822.805747:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574822.805750:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574822.805756:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574822.805763:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805767:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574822.805771:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805774:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574822.805778:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574822.805784:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805790:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574822.805793:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574822.805800:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574822.805805:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574822.805809:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574822.805813:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574822.805820:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574822.805823:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574822.805829:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574822.805835:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574822.805842:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574822.805845:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574822.805850:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574822.805853:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574822.805856:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574822.805863:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574822.805869:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574822.805879:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393163392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574822.805890:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574822.805893:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574822.805899:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393163392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574822.805909:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574822.805911:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574822.805914:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574822.805917:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574822.805920:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574822.805925:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574822.805930:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11753
00000400:00000200:0.0:1788574822.805938:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574822.805946:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574822.805952:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574822.805958:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805963:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574822.805969:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.805973:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574822.805977:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574822.805979:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574822.805983:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574822.805986:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574822.805991:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574822.805996:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.1:1788574822.806000:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788574822.806003:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.1:1788574822.806004:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1788574822.806008:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574822.806013:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574822.806018:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574822.806022:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574822.806029:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574822.806034:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574822.806037:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574822.806041:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574822.806043:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574822.806045:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574822.806048:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574822.806053:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574822.806056:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574822.806060:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788574822.806066:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574822.806072:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574822.806075:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574822.806080:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574822.806082:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574822.806085:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574822.806090:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574822.806095:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574822.806105:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393163520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574822.806115:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574822.806119:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574822.806125:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393163520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574822.806140:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574822.806144:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574822.806150:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574822.806159:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11764)
00000800:00000200:3.1:1788574822.806356:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574822.806358:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788574822.806548:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574822.806549:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1788574823.317485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574823.317492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574823.317493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574823.317497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574823.317500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574823.317503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574823.317505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574823.317507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574823.317509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574823.765499:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306427321, at 4306427328
00000100:00000040:0.0:1788574823.765503:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306427321, at 4306427328
00000100:00000010:1.0:1788574823.765515:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:0.0:1788574823.765517:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:0.0:1788574824.341492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574824.341499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574824.341500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574824.341504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574824.341507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574824.341510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574824.341512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574824.341514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574824.341516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574827.925508:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574827.925520:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000800:00000200:0.0:1788574827.925521:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000100:00000001:1.0:1788574827.925522:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574827.925527:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000800:00000200:0.0:1788574827.925528:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000100:00000001:1.0:1788574827.925531:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:0.0:1788574827.925534:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000100:00080000:1.0:1788574827.925537:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11759
00000400:00000200:1.0:1788574827.925546:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574827.925552:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574827.925555:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574827.925558:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925561:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574827.925563:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925566:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574827.925568:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574827.925570:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574827.925572:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574827.925574:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574827.925576:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574827.925577:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574827.925579:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925581:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574827.925582:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925584:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574827.925586:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574827.925588:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925590:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574827.925592:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574827.925594:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574827.925596:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574827.925598:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574827.925600:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574827.925606:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788574827.925609:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574827.925613:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574827.925615:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574827.925617:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574827.925620:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574827.925623:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574827.925630:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788574827.925635:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574827.925643:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393163648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574827.925652:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574827.925655:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788574827.925659:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000040:1.0:1788574827.925660:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393163648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000001:0.0:1788574827.925664:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574827.925675:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788574827.925679:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000001:1.0:1788574827.925681:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000001:0.0:1788574827.925682:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1788574827.925687:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574827.925690:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574827.925692:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574827.925693:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574827.925695:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574827.925698:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11759
00000400:00000200:1.0:1788574827.925700:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574827.925703:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574827.925705:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574827.925707:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925709:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574827.925711:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925713:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574827.925715:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574827.925716:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574827.925717:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574827.925720:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574827.925721:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574827.925723:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574827.925726:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925727:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574827.925729:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925731:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574827.925733:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574827.925736:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925739:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574827.925741:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574827.925747:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574827.925751:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574827.925753:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574827.925766:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574827.925771:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574827.925774:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574827.925779:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788574827.925786:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574827.925790:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574827.925792:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574827.925796:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574827.925797:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574827.925800:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574827.925805:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788574827.925811:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574827.925821:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393163776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574827.925829:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574827.925831:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574827.925835:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393163776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574827.925855:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574827.925860:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574827.925868:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574827.925870:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574827.925871:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574827.925872:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574827.925875:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11759
00000400:00000200:1.0:1788574827.925878:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574827.925884:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574827.925886:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574827.925891:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925896:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574827.925902:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925909:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574827.925913:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574827.925915:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574827.925919:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574827.925923:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574827.925929:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574827.925934:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574827.925942:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925948:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574827.925957:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925963:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574827.925970:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574827.925978:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574827.925985:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574827.925991:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574827.925997:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574827.926002:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574827.926006:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574827.926009:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:3.1:1788574827.926013:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000008:00000001:1.0:1788574827.926014:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574827.926019:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788574827.926020:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574827.926027:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788574827.926032:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574827.926036:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574827.926039:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574827.926041:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574827.926044:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574827.926048:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574827.926052:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574827.926057:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574827.926070:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393163904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574827.926092:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574827.926095:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574827.926101:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393163904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574827.926131:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574827.926137:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574827.926142:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574827.926147:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11769)
00000800:00000200:3.1:1788574827.926424:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574827.926425:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788574827.926640:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574827.926642:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:1.0:1788574828.885466:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306432441, at 4306432448
00000100:00000040:0.0:1788574828.885469:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306432441, at 4306432448
00000100:00000010:1.0:1788574828.885481:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788574828.885484:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00080000:1.0:1788574833.045488:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574833.045496:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574833.045500:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574833.045503:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574833.045507:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574833.045513:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11764
00000400:00000200:1.0:1788574833.045521:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574833.045526:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574833.045528:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574833.045531:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045533:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574833.045535:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045537:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574833.045540:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574833.045541:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574833.045543:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574833.045546:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574833.045547:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574833.045549:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574833.045551:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045552:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574833.045553:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045554:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574833.045557:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574833.045559:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045561:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574833.045564:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574833.045566:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574833.045568:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574833.045570:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574833.045571:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574833.045580:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788574833.045582:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574833.045585:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574833.045586:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574833.045589:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574833.045591:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574833.045594:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574833.045600:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788574833.045605:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574833.045613:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393164032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574833.045620:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574833.045623:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574833.045627:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393164032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574833.045644:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574833.045646:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574833.045649:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574833.045650:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574833.045651:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574833.045653:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574833.045655:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11764
00000400:00000200:1.0:1788574833.045658:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574833.045659:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574833.045661:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574833.045663:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045665:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574833.045666:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045668:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574833.045670:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574833.045671:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574833.045672:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574833.045674:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574833.045676:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574833.045678:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574833.045680:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045681:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574833.045684:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045686:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574833.045689:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574833.045691:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045695:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574833.045697:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574833.045704:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574833.045707:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574833.045708:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574833.045711:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574833.045713:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574833.045717:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574833.045722:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788574833.045726:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574833.045729:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574833.045733:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574833.045738:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574833.045740:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574833.045744:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574833.045752:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788574833.045757:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574833.045766:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393164160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574833.045774:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574833.045777:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574833.045783:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393164160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574833.045810:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574833.045816:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574833.045819:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574833.045820:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574833.045823:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574833.045827:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574833.045831:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11764
00000400:00000200:1.0:1788574833.045838:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574833.045842:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574833.045847:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574833.045853:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045857:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574833.045863:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045866:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574833.045870:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574833.045872:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574833.045875:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574833.045876:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574833.045881:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574833.045884:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574833.045888:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045892:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574833.045896:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045898:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574833.045901:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574833.045907:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574833.045909:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574833.045913:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574833.045917:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574833.045919:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574833.045922:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574833.045926:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574833.045931:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574833.045933:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574833.045940:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788574833.045945:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574833.045951:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574833.045955:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574833.045960:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574833.045963:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574833.045967:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574833.045977:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000800:00000200:3.1:1788574833.045982:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574833.045984:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574833.045985:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00100000:1.0:1788574833.045996:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393164288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574833.046012:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574833.046016:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574833.046025:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393164288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574833.046077:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574833.046084:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574833.046090:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574833.046097:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11774)
00000800:00000200:3.1:1788574833.046360:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574833.046362:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788574833.046571:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574833.046573:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1788574834.005509:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306437561, at 4306437568
00000100:00000040:1.0:1788574834.005509:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306437561, at 4306437568
00000100:00000010:0.0:1788574834.005524:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:1.0:1788574834.005527:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00080000:1.0:1788574838.165512:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574838.165521:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574838.165524:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574838.165527:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574838.165531:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574838.165537:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11769
00000400:00000200:1.0:1788574838.165545:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574838.165551:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574838.165554:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574838.165557:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165560:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574838.165562:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165564:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574838.165568:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574838.165569:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574838.165571:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574838.165574:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574838.165576:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574838.165578:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574838.165581:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165582:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574838.165584:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165585:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574838.165587:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574838.165589:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165591:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574838.165593:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574838.165596:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574838.165598:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574838.165599:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574838.165601:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574838.165608:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788574838.165611:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574838.165614:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574838.165615:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574838.165618:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574838.165620:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574838.165623:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574838.165629:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788574838.165635:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574838.165642:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393164416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574838.165650:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574838.165653:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574838.165657:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393164416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574838.165673:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574838.165677:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574838.165680:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574838.165682:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574838.165683:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574838.165686:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574838.165690:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11769
00000400:00000200:1.0:1788574838.165702:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574838.165706:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574838.165709:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574838.165712:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165714:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574838.165716:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165718:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574838.165721:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574838.165722:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574838.165724:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574838.165726:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574838.165729:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574838.165731:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574838.165735:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165737:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574838.165740:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165741:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574838.165744:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574838.165746:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165749:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574838.165751:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574838.165756:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574838.165758:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574838.165761:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574838.165764:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574838.165767:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574838.165769:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574838.165773:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788574838.165775:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574838.165778:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574838.165779:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574838.165781:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574838.165782:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574838.165784:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574838.165787:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788574838.165790:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574838.165795:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393164544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574838.165803:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574838.165806:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574838.165810:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393164544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574838.165825:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574838.165828:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574838.165832:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574838.165833:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574838.165834:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574838.165837:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574838.165839:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11769
00000400:00000200:1.0:1788574838.165843:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574838.165845:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574838.165848:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574838.165850:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165852:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574838.165855:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165857:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574838.165859:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574838.165861:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574838.165862:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574838.165864:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574838.165866:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574838.165868:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574838.165871:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165873:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574838.165875:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165877:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574838.165880:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574838.165883:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574838.165885:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574838.165887:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574838.165890:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574838.165892:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574838.165894:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574838.165896:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574838.165898:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574838.165900:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574838.165903:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788574838.165906:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574838.165908:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574838.165910:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574838.165910:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574838.165911:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574838.165913:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.1:1788574838.165914:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
02000000:00000001:1.0:1788574838.165915:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574838.165933:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574838.165937:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574838.165942:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393164672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574838.165950:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574838.165952:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574838.165955:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393164672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574838.165962:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574838.165966:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574838.165970:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574838.165976:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11779)
00000800:00000200:3.1:1788574838.166274:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574838.166276:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788574838.166561:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574838.166564:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:0.0:1788574839.125477:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306442681, at 4306442688
00000100:00000040:1.0:1788574839.125477:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306442681, at 4306442688
00000100:00000010:1.0:1788574839.125492:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:0.0:1788574839.125495:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00080000:0.0:1788574843.285546:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574843.285561:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574843.285564:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574843.285570:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574843.285575:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574843.285584:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11774
00000400:00000200:0.0:1788574843.285594:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574843.285601:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574843.285605:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574843.285608:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285612:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574843.285615:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285618:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574843.285622:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574843.285624:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574843.285627:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574843.285631:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574843.285634:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574843.285636:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574843.285639:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285641:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574843.285643:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285645:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574843.285648:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574843.285653:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285655:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574843.285658:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574843.285662:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574843.285665:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574843.285668:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574843.285672:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574843.285685:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788574843.285689:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574843.285694:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574843.285696:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574843.285699:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574843.285703:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574843.285707:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574843.285722:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788574843.285731:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574843.285743:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393164800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574843.285753:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574843.285758:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574843.285763:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393164800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574843.285787:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574843.285801:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574843.285805:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574843.285806:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574843.285808:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574843.285811:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574843.285814:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11774
00000400:00000200:0.0:1788574843.285817:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574843.285820:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574843.285823:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574843.285826:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285836:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574843.285838:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285843:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574843.285846:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574843.285849:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574843.285852:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574843.285855:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574843.285860:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574843.285863:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574843.285870:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285875:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574843.285879:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285888:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574843.285894:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574843.285901:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574843.285906:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574843.285911:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574843.285919:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574843.285923:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574843.285928:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574843.285935:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574843.285950:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574843.285955:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574843.285962:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574843.285964:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574843.285967:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574843.285970:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574843.285972:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574843.285974:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574843.285976:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574843.285985:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574843.285992:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574843.286001:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393164928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574843.286017:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574843.286022:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574843.286032:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393164928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574843.286043:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574843.286048:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574843.286052:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574843.286054:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574843.286056:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574843.286061:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574843.286066:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11774
00000400:00000200:0.0:1788574843.286075:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574843.286081:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574843.286087:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574843.286093:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.286097:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574843.286103:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.286108:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574843.286113:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574843.286116:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574843.286120:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574843.286123:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574843.286127:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574843.286129:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574843.286137:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.286142:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574843.286148:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574843.286151:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574843.286156:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574843.286163:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574843.286166:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574843.286171:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574843.286174:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574843.286178:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574843.286182:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.1:1788574843.286186:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000008:00100000:0.0:1788574843.286187:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574843.286193:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574843.286195:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574843.286200:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788574843.286202:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574843.286205:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574843.286206:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574843.286209:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574843.286212:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574843.286214:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574843.286221:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574843.286227:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574843.286234:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393165056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574843.286244:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574843.286248:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574843.286252:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393165056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574843.286266:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574843.286271:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574843.286277:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574843.286293:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11784)
00000800:00000200:3.1:1788574843.286467:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574843.286601:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788574844.245524:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306447801, at 4306447808
00000100:00000040:1.0:1788574844.245524:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306447801, at 4306447808
00000100:00000010:0.0:1788574844.245542:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788574844.245544:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00080000:0.0:1788574848.405486:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574848.405495:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574848.405497:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574848.405500:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574848.405503:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574848.405509:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11779
00000400:00000200:0.0:1788574848.405516:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574848.405521:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574848.405524:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574848.405527:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405529:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574848.405532:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405534:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574848.405537:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574848.405539:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574848.405541:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574848.405544:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574848.405546:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574848.405548:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574848.405550:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405552:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574848.405553:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405555:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574848.405557:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574848.405559:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405561:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574848.405563:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574848.405565:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574848.405567:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574848.405569:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574848.405571:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574848.405576:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788574848.405578:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574848.405582:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574848.405583:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574848.405585:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574848.405587:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574848.405591:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574848.405596:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574848.405601:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574848.405609:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393165184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574848.405616:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574848.405620:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574848.405623:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393165184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574848.405646:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574848.405650:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574848.405653:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574848.405655:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574848.405657:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574848.405660:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574848.405678:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11779
00000400:00000200:0.0:1788574848.405682:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574848.405685:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574848.405687:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574848.405690:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405692:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574848.405693:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405695:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574848.405697:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574848.405698:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574848.405700:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574848.405702:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574848.405705:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574848.405707:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574848.405709:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405712:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574848.405714:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405716:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574848.405718:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574848.405721:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405724:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574848.405725:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574848.405731:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574848.405733:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574848.405735:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574848.405739:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574848.405742:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574848.405744:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574848.405747:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788574848.405750:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574848.405752:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574848.405754:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574848.405755:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574848.405757:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574848.405758:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574848.405762:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574848.405766:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574848.405771:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393165312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574848.405778:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574848.405780:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574848.405784:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393165312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574848.405797:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574848.405800:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574848.405803:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574848.405805:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574848.405807:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574848.405809:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574848.405812:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11779
00000400:00000200:0.0:1788574848.405816:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574848.405820:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574848.405823:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574848.405826:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405829:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574848.405832:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405834:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574848.405836:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574848.405838:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574848.405839:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574848.405841:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574848.405844:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574848.405847:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574848.405852:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405856:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574848.405861:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405866:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574848.405870:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574848.405878:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574848.405885:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574848.405891:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574848.405895:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574848.405900:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574848.405903:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574848.405906:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574848.405911:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574848.405917:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574848.405925:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788574848.405930:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574848.405935:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574848.405939:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574848.405941:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574848.405943:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574848.405947:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574848.405953:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574848.405960:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574848.405967:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393165440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574848.405979:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574848.405984:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574848.405990:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393165440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574848.406001:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574848.406005:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574848.406010:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574848.406020:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11789)
00000800:00000200:3.1:1788574848.406149:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574848.406153:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788574848.406786:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574848.406789:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788574848.407197:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574848.407198:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:0.0:1788574849.365488:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306452921, at 4306452928
00000100:00000040:1.0:1788574849.365488:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306452921, at 4306452928
00000100:00000010:1.0:1788574849.365503:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000010:0.0:1788574849.365505:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00080000:1.0:1788574853.525484:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574853.525493:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574853.525495:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574853.525498:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574853.525502:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574853.525510:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11784
00000400:00000200:1.0:1788574853.525519:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574853.525526:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574853.525530:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574853.525533:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525546:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574853.525550:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525552:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574853.525556:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574853.525558:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574853.525561:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574853.525564:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574853.525567:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574853.525569:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574853.525572:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525579:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574853.525581:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525584:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574853.525587:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574853.525590:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525593:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574853.525596:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574853.525599:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574853.525602:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574853.525605:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574853.525607:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574853.525617:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788574853.525621:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574853.525625:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574853.525626:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574853.525629:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574853.525632:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574853.525636:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574853.525644:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574853.525650:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574853.525660:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393165568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574853.525671:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574853.525676:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574853.525681:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393165568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574853.525702:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574853.525706:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574853.525710:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574853.525711:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574853.525713:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574853.525715:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574853.525717:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11784
00000400:00000200:1.0:1788574853.525721:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574853.525723:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574853.525725:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574853.525727:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525729:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574853.525731:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525733:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574853.525736:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574853.525737:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574853.525739:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574853.525741:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574853.525742:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574853.525744:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574853.525746:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525747:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574853.525749:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525750:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574853.525755:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574853.525758:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525762:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574853.525764:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574853.525771:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574853.525775:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574853.525777:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574853.525782:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574853.525787:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574853.525790:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574853.525796:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788574853.525799:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574853.525803:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574853.525806:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574853.525809:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574853.525810:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574853.525812:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574853.525817:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574853.525823:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574853.525832:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393165696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574853.525855:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574853.525860:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574853.525865:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393165696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574853.525877:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574853.525882:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574853.525884:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574853.525886:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574853.525888:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574853.525892:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574853.525896:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11784
00000400:00000200:1.0:1788574853.525901:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574853.525904:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574853.525905:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574853.525908:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525913:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574853.525915:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525916:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574853.525918:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574853.525920:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574853.525923:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574853.525926:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574853.525931:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574853.525937:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574853.525944:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525947:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574853.525953:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525959:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574853.525965:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574853.525974:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574853.525981:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574853.525987:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574853.525991:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574853.525995:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574853.525997:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574853.525999:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574853.526003:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574853.526008:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574853.526012:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788574853.526014:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574853.526017:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574853.526019:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574853.526022:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574853.526023:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574853.526025:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574853.526028:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788574853.526033:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574853.526044:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393165824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574853.526063:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574853.526069:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574853.526079:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393165824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574853.526090:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574853.526094:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574853.526100:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574853.526105:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11794)
00000800:00000200:3.1:1788574853.526136:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574853.526140:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788574853.526627:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574853.526629:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788574853.526940:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574853.526943:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:1.0:1788574854.485546:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306458041, at 4306458048
00000100:00000040:0.0:1788574854.485561:0:216334:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306458041, at 4306458048
00000100:00000010:1.0:1788574854.485563:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788574854.485579:0:216334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00080000:1.0:1788574858.645490:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574858.645499:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574858.645501:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574858.645505:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574858.645508:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574858.645514:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11789
00000400:00000200:1.0:1788574858.645524:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574858.645530:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574858.645532:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574858.645535:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645538:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574858.645540:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645542:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574858.645545:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574858.645547:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574858.645549:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574858.645551:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574858.645553:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574858.645555:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574858.645557:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645558:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574858.645560:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645561:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574858.645563:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574858.645565:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645567:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574858.645569:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574858.645572:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574858.645574:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574858.645575:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574858.645577:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574858.645584:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788574858.645587:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574858.645590:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574858.645591:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574858.645593:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574858.645596:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574858.645599:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574858.645605:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574858.645610:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574858.645617:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393165952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574858.645625:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574858.645628:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574858.645632:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393165952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574858.645648:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574858.645651:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574858.645653:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574858.645654:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574858.645655:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574858.645658:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574858.645659:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11789
00000400:00000200:1.0:1788574858.645661:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574858.645663:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574858.645665:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574858.645666:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645668:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574858.645670:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645672:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574858.645674:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574858.645675:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574858.645676:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574858.645678:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574858.645680:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574858.645682:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574858.645686:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645691:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574858.645695:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645701:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574858.645708:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574858.645714:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645721:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574858.645727:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574858.645735:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574858.645740:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574858.645743:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574858.645748:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574858.645753:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574858.645757:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574858.645761:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788574858.645766:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574858.645772:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574858.645773:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574858.645777:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574858.645778:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574858.645780:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574858.645785:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574858.645789:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574858.645797:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393166080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574858.645810:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574858.645815:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574858.645824:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393166080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574858.645850:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574858.645857:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574858.645862:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574858.645865:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574858.645868:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574858.645873:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574858.645879:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11789
00000400:00000200:1.0:1788574858.645884:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574858.645891:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574858.645894:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574858.645899:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645902:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574858.645907:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645909:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574858.645911:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574858.645914:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574858.645916:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574858.645920:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574858.645922:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574858.645924:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574858.645929:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645931:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574858.645936:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645937:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574858.645939:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574858.645945:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574858.645949:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574858.645951:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574858.645955:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574858.645958:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574858.645961:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574858.645965:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574858.645968:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574858.645971:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574858.645976:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788574858.645980:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574858.645985:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574858.645987:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574858.645992:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574858.645994:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574858.645997:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574858.646001:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574858.646005:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574858.646012:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393166208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574858.646030:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574858.646033:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574858.646039:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393166208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574858.646046:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574858.646048:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574858.646053:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574858.646058:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11799)
00000800:00000200:3.1:1788574858.646069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574858.646073:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788574858.646469:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574858.646472:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788574858.646686:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574858.646688:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:0.0:1788574859.605490:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306463161, at 4306463168
00000100:00000040:1.0:1788574859.605490:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306463161, at 4306463168
00000100:00000010:1.0:1788574859.605505:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788574859.605508:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:0.0:1788574862.229472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574862.229478:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574862.229480:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574862.229484:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574862.229487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574862.229490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574862.229492:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574862.229494:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574862.229496:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574863.253458:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574863.253464:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574863.253466:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574863.253470:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574863.253474:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574863.253477:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574863.253479:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574863.253485:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574863.253488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574863.766067:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574863.766073:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788574863.766449:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574863.766450:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788574863.766659:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574863.766662:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:0.0:1788574864.277507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574864.277514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574864.277515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574864.277520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574864.277523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574864.277526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574864.277528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574864.277530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574864.277532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574864.725498:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306468281, at 4306468288
00000100:00000040:1.0:1788574864.725498:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306468281, at 4306468288
00000100:00000010:0.0:1788574864.725511:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:1.0:1788574864.725514:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:0.0:1788574865.301582:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574865.301589:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574865.301593:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574865.301597:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574865.301602:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574865.301605:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574865.301607:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574865.301610:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574865.301614:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788574865.813635:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574865.813647:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574865.813654:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574865.813863:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574865.813874:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574865.813884:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574865.813893:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574865.813897:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574866.325522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574866.325531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574866.325533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574866.325538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574866.325541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574866.325546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574866.325549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574866.325552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574866.325556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574867.349520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574867.349529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574867.349531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574867.349537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574867.349541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574867.349545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574867.349547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574867.349550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574867.349553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574868.373527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574868.373536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574868.373538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574868.373545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574868.373549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574868.373555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574868.373558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574868.373561:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574868.373563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574868.886065:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574868.886072:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788574868.886491:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574868.886493:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788574868.886754:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574868.886757:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:0.0:1788574869.397493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574869.397499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574869.397501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574869.397505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574869.397507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574869.397510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574869.397512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574869.397514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574869.397516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574869.845499:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306473401, at 4306473408
00000100:00000040:1.0:1788574869.845499:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306473401, at 4306473408
00000100:00000010:0.0:1788574869.845515:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:1.0:1788574869.845518:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000001:0.0:1788574870.421511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574870.421531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574870.421534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574870.421540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574870.421544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574870.421549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574870.421553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574870.421556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574870.421560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574871.445650:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574871.445659:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574871.445662:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574871.445668:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574871.445672:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574871.445676:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574871.445678:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574871.445682:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574871.445686:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574872.469633:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574872.469641:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574872.469644:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574872.469652:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574872.469656:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574872.469661:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574872.469664:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574872.469667:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574872.469671:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574873.493517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574873.493524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574873.493527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574873.493534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574873.493538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574873.493542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574873.493545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574873.493548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574873.493552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574874.006113:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574874.006116:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788574874.006620:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574874.006623:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788574874.006938:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574874.006941:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:1.0:1788574874.517519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574874.517525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574874.517527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574874.517531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574874.517534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574874.517537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574874.517539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574874.517541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574874.517544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574874.965503:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306478521, at 4306478528
00000100:00000040:1.0:1788574874.965503:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306478521, at 4306478528
00000100:00000010:0.0:1788574874.965518:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000100:00000010:1.0:1788574874.965520:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00000100:00000001:1.0:1788574875.541564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574875.541576:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574875.541579:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574875.541587:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574875.541591:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574875.541596:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574875.541599:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574875.541603:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574875.541607:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574876.565552:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574876.565564:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574876.565567:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574876.565575:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574876.565580:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574876.565584:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574876.565588:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574876.565591:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574876.565594:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574879.126207:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574879.126210:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574879.126599:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574879.126601:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788574879.126810:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574879.126812:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:0.0:1788574880.085546:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306483641, at 4306483648
00000100:00000040:1.0:1788574880.085550:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306483641, at 4306483648
00000100:00000010:0.0:1788574880.085566:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00000100:00000010:1.0:1788574880.085569:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000100:00080000:0.0:1788574884.245496:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574884.245510:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574884.245513:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574884.245517:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574884.245521:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574884.245527:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11815
00000400:00000200:0.0:1788574884.245535:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574884.245541:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574884.245543:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574884.245546:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245548:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574884.245550:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245552:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574884.245555:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574884.245557:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574884.245559:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574884.245561:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574884.245563:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574884.245564:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574884.245566:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245568:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574884.245569:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245570:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574884.245573:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574884.245575:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245577:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574884.245579:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574884.245582:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574884.245584:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574884.245586:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574884.245587:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574884.245596:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788574884.245601:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574884.245605:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574884.245607:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574884.245610:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574884.245613:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574884.245618:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574884.245626:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788574884.245633:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574884.245644:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393167872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574884.245655:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574884.245658:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574884.245662:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393167872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574884.245684:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574884.245688:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574884.245692:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574884.245694:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574884.245696:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574884.245699:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574884.245702:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11815
00000400:00000200:0.0:1788574884.245707:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574884.245711:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574884.245714:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574884.245718:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245720:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574884.245723:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245725:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574884.245728:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574884.245733:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574884.245735:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574884.245740:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574884.245745:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574884.245749:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574884.245760:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245765:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574884.245772:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245778:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574884.245786:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574884.245794:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245797:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574884.245799:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574884.245806:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574884.245810:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574884.245815:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574884.245821:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574884.245826:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574884.245829:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574884.245835:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788574884.245837:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574884.245839:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574884.245840:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574884.245841:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574884.245846:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574884.245847:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574884.245852:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574884.245859:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574884.245868:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393168000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574884.245898:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574884.245899:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574884.245906:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393168000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574884.245928:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574884.245933:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574884.245937:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574884.245938:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574884.245939:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574884.245941:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574884.245947:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11815
00000400:00000200:0.0:1788574884.245955:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574884.245961:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574884.245966:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574884.245973:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245979:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574884.245986:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.245990:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574884.245994:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574884.245996:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574884.245999:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574884.246001:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574884.246003:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574884.246008:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574884.246014:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.246019:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574884.246026:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574884.246031:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574884.246046:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:2.1:1788574884.246052:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574884.246058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00080000:0.0:1788574884.246059:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574884.246066:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574884.246071:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574884.246077:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574884.246080:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574884.246083:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574884.246086:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574884.246090:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574884.246093:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574884.246100:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788574884.246103:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574884.246106:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574884.246109:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574884.246111:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574884.246114:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574884.246116:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574884.246120:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788574884.246126:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574884.246136:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393168128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574884.246153:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574884.246157:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574884.246163:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393168128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574884.246193:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574884.246196:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574884.246202:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574884.246208:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11825)
00000800:00000200:2.1:1788574884.246503:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574884.246505:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788574884.246701:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574884.246703:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:0.0:1788574885.205478:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306488761, at 4306488768
00000100:00000040:1.0:1788574885.205479:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306488761, at 4306488768
00000100:00000010:0.0:1788574885.205493:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000100:00000010:1.0:1788574885.205496:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000100:00080000:1.0:1788574889.365537:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574889.365548:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574889.365551:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574889.365555:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574889.365560:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574889.365569:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11820
00000400:00000200:1.0:1788574889.365579:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574889.365586:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574889.365589:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574889.365594:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365598:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574889.365600:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365603:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574889.365607:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574889.365609:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574889.365612:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574889.365615:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574889.365618:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574889.365621:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574889.365623:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365625:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574889.365628:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365629:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574889.365632:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574889.365635:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365639:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574889.365641:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574889.365645:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574889.365648:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574889.365650:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574889.365652:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574889.365663:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788574889.365666:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574889.365670:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574889.365672:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574889.365675:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574889.365678:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574889.365682:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574889.365690:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574889.365697:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574889.365706:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393168256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574889.365715:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574889.365719:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574889.365723:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393168256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574889.365746:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574889.365749:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574889.365753:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574889.365754:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574889.365755:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574889.365757:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574889.365759:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11820
00000400:00000200:1.0:1788574889.365763:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574889.365766:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574889.365768:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574889.365771:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365774:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574889.365777:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365780:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574889.365784:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574889.365786:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574889.365788:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574889.365794:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574889.365802:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574889.365808:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574889.365821:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365830:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574889.365851:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365862:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574889.365867:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574889.365874:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574889.365880:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574889.365889:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574889.365900:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574889.365906:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574889.365915:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574889.365923:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574889.365933:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574889.365943:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574889.365957:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788574889.365972:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574889.365982:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574889.365986:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574889.365991:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574889.365994:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574889.365998:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574889.366007:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788574889.366016:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574889.366044:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393168384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574889.366076:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574889.366083:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574889.366092:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393168384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574889.366117:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574889.366120:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574889.366126:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574889.366128:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574889.366134:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574889.366140:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574889.366147:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11820
00000400:00000200:1.0:1788574889.366157:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574889.366170:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574889.366178:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574889.366189:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.366199:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574889.366209:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.366220:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574889.366232:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574889.366237:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574889.366242:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574889.366247:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574889.366256:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574889.366262:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574889.366273:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.366277:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574889.366284:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574889.366292:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574889.366302:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574889.366310:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574889.366317:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574889.366319:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1788574889.366322:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:2.1:1788574889.366326:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:1.0:1788574889.366329:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574889.366335:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574889.366340:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574889.366346:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574889.366353:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574889.366357:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574889.366367:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788574889.366376:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574889.366381:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574889.366386:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574889.366419:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574889.366423:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574889.366426:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574889.366433:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574889.366442:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574889.366457:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393168512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574889.366524:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574889.366531:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574889.366545:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393168512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574889.366565:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574889.366570:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574889.366582:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574889.366590:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11830)
00000800:00000200:2.1:1788574889.366835:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574889.366838:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788574889.367058:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574889.367060:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:0.0:1788574890.325489:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306493882, at 4306493888
00000100:00000040:1.0:1788574890.325496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306493882, at 4306493888
00000100:00000010:1.0:1788574890.325513:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000100:00000010:0.0:1788574890.325516:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000100:00000001:1.0:1788574891.925571:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574891.925578:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574891.925580:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574891.925587:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574891.925592:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574891.925595:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574891.925597:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574891.925599:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574891.925601:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574892.949496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574892.949502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574892.949504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574892.949507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574892.949510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574892.949514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574892.949516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574892.949517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574892.949520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574893.973569:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574893.973579:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574893.973581:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574893.973589:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574893.973596:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574893.973601:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574893.973604:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574893.973608:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574893.973612:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574894.485548:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574894.485560:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574894.485563:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574894.485567:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574894.485572:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574894.485580:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11825
00000400:00000200:1.0:1788574894.485590:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574894.485598:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574894.485601:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574894.485604:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485608:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574894.485611:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485613:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574894.485617:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574894.485619:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574894.485622:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574894.485625:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574894.485627:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574894.485628:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574894.485630:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485632:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574894.485633:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485635:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574894.485637:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574894.485640:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485642:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574894.485645:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574894.485649:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574894.485652:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574894.485654:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574894.485656:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574894.485667:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788574894.485670:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574894.485674:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574894.485676:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574894.485679:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574894.485683:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574894.485687:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574894.485696:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788574894.485703:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574894.485712:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393168640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574894.485721:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574894.485725:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574894.485729:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393168640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574894.485757:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574894.485761:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574894.485766:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574894.485768:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574894.485770:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574894.485773:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574894.485777:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11825
00000400:00000200:1.0:1788574894.485781:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574894.485785:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574894.485788:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574894.485792:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485794:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574894.485797:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485798:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574894.485814:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574894.485815:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574894.485818:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574894.485820:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574894.485824:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574894.485830:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574894.485836:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485841:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574894.485847:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485851:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574894.485857:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574894.485863:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574894.485868:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574894.485872:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574894.485880:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574894.485884:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574894.485887:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574894.485892:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574894.485895:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574894.485900:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574894.485904:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788574894.485909:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574894.485913:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574894.485915:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574894.485918:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574894.485921:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574894.485925:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574894.485930:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788574894.485936:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574894.485946:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393168768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574894.485964:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574894.485970:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574894.485978:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393168768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574894.485996:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574894.485999:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574894.486003:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574894.486007:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574894.486010:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574894.486018:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574894.486028:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11825
00000400:00000200:1.0:1788574894.486040:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574894.486048:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574894.486053:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574894.486058:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.486062:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574894.486067:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.486070:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574894.486072:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574894.486074:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574894.486076:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574894.486079:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574894.486084:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574894.486086:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574894.486090:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.486095:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574894.486097:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574894.486100:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574894.486104:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574894.486109:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574894.486114:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574894.486119:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574894.486124:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574894.486129:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574894.486135:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574894.486141:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574894.486146:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574894.486149:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574894.486157:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788574894.486164:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574894.486168:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574894.486170:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574894.486172:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574894.486174:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574894.486177:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574894.486184:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788574894.486191:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574894.486197:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393168896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574894.486208:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574894.486213:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574894.486218:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788574894.486221:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393168896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788574894.486223:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788574894.486240:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574894.486243:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574894.486249:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574894.486260:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11835)
00000800:00000200:2.1:1788574894.486663:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574894.486665:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788574894.486916:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574894.486918:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788574894.997498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574894.997505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574894.997508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574894.997514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574894.997517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574894.997521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574894.997524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574894.997526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574894.997529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574895.445546:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306499001, at 4306499008
00000100:00000040:1.0:1788574895.445546:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306499001, at 4306499008
00000100:00000010:0.0:1788574895.445569:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000100:00000010:1.0:1788574895.445571:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000100:00000001:1.0:1788574896.021629:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574896.021640:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574896.021642:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574896.021648:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574896.021651:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574896.021655:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574896.021657:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574896.021660:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574896.021663:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574897.045479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574897.045487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574897.045489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574897.045494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574897.045498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574897.045502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574897.045504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574897.045507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574897.045509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574898.069582:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574898.069592:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574898.069595:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574898.069602:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574898.069608:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574898.069613:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574898.069616:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574898.069620:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574898.069624:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574899.093581:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574899.093589:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574899.093593:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574899.093598:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574899.093603:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574899.093608:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574899.093612:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574899.093615:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574899.093619:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574899.606231:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574899.606237:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788574899.606663:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574899.606665:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788574899.606921:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574899.606923:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:1.0:1788574900.117551:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574900.117558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574900.117561:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574900.117567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574900.117572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574900.117576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574900.117579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574900.117582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574900.117586:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574900.565511:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306504121, at 4306504128
00000100:00000040:1.0:1788574900.565511:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306504121, at 4306504128
00000100:00000010:0.0:1788574900.565527:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:1.0:1788574900.565532:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:1.0:1788574901.141498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574901.141504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574901.141506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574901.141510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574901.141512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574901.141515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574901.141518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574901.141520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574901.141522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574902.165507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574902.165515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574902.165517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574902.165523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574902.165526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574902.165530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574902.165532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574902.165535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574902.165538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574903.189494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574903.189500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574903.189502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574903.189506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574903.189509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574903.189511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574903.189513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574903.189515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574903.189518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574904.213518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574904.213526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574904.213529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574904.213535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574904.213538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574904.213540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574904.213543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574904.213544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574904.213546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574904.726119:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574904.726122:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574904.726496:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574904.726498:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574904.726685:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574904.726686:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:1.0:1788574905.237520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574905.237528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574905.237531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574905.237536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574905.237540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574905.237545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574905.237548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574905.237551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574905.237554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574905.685492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306509241, at 4306509248
00000100:00000040:0.0:1788574905.685497:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306509241, at 4306509248
00000100:00000010:1.0:1788574905.685506:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000100:00000010:0.0:1788574905.685509:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000800:00000200:0.0:1788574905.749482:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788574905.749488:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788574905.749492:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788574905.749639:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574905.749655:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:1.0:1788574905.749658:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788574905.749660:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574905.749666:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574906.261475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574906.261480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574906.261482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574906.261486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574906.261489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574906.261492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574906.261495:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574906.261499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574906.261503:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574907.285521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574907.285530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574907.285533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574907.285538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574907.285542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574907.285546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574907.285549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574907.285552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574907.285555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574908.309489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574908.309498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574908.309500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574908.309505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574908.309509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574908.309512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574908.309515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574908.309517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574908.309520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574909.333475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574909.333481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574909.333483:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574909.333488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574909.333491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574909.333494:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574909.333495:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574909.333497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574909.333499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574909.845972:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574909.846213:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574909.846325:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788574910.357494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574910.357500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574910.357502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574910.357507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574910.357509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574910.357513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574910.357515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574910.357517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574910.357519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574910.805492:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306514361, at 4306514368
00000100:00000040:1.0:1788574910.805492:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306514361, at 4306514368
00000100:00000010:0.0:1788574910.805507:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000100:00000010:1.0:1788574910.805509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00000100:00000001:1.0:1788574911.381512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574911.381518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574911.381520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574911.381525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574911.381527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574911.381531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574911.381533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574911.381535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574911.381537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574912.405549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574912.405557:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574912.405560:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574912.405565:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574912.405569:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574912.405573:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574912.405576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574912.405579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574912.405581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788574912.917775:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788574912.917828:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788574912.917839:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574912.917845:0:2351:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788574912.917851:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574912.917856:0:2351:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788574912.917863:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574912.917868:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:1.0:1788574912.917875:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574912.917880:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574912.917889:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c652a8
00000400:00000010:1.0:1788574912.917893:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c652a8.
00000100:00000001:1.0:1788574912.917899:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574912.917913:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2300 x1875444393169792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574912.917955:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574912.917961:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393169792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574912.917969:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574912.917971:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574912.917984:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574912.917988:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133b00.
00000400:00000200:1.0:1788574912.917991:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574912.917995:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574912.917999:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:1.0:1788574912.918002:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:1.0:1788574912.918006:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574912.918012:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0a80 x1875444393169920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574912.918022:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574912.918026:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393169920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574912.918033:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574912.918038:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574912.918046:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574912.918049:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:1.0:1788574912.918055:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574912.918065:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574912.918069:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65a18
00000400:00000010:1.0:1788574912.918072:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65a18.
00000100:00000001:1.0:1788574912.918079:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574912.918095:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e1880 x1875444393170048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574912.918109:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574912.918117:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393170048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574912.918160:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574912.918167:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788574913.429544:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574913.429556:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574913.429559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574913.429567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574913.429572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574913.429577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574913.429580:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574913.429584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574913.429588:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574914.453512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574914.453526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574914.453528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574914.453534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574914.453537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574914.453541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574914.453544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574914.453547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574914.453550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574914.966146:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574914.966154:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788574914.966754:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574914.966756:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788574914.967050:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574914.967053:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:1.0:1788574915.477508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574915.477515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574915.477517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574915.477523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574915.477526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574915.477530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574915.477533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574915.477535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574915.477538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574915.925494:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306519481, at 4306519488
00000100:00000040:1.0:1788574915.925494:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306519481, at 4306519488
00000100:00000010:1.0:1788574915.925509:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000100:00000010:0.0:1788574915.925511:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00000100:00000001:1.0:1788574916.501473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574916.501479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574916.501481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574916.501485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574916.501488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574916.501491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574916.501493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574916.501496:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574916.501498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574917.525498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574917.525506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574917.525508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574917.525514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574917.525518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574917.525522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574917.525525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574917.525528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574917.525531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574918.549469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574918.549475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574918.549477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574918.549481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574918.549484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574918.549487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574918.549489:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574918.549491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574918.549494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574919.573517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574919.573524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574919.573527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574919.573533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574919.573536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574919.573539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574919.573541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574919.573544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574919.573546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574920.086065:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574920.086071:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788574920.086451:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574920.086453:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788574920.086676:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574920.086677:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1788574920.597463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574920.597470:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574920.597472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574920.597477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574920.597480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574920.597483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574920.597486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574920.597488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574920.597490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574921.045496:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306524601, at 4306524608
00000100:00000040:1.0:1788574921.045496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306524601, at 4306524608
00000100:00000010:0.0:1788574921.045510:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000100:00000010:1.0:1788574921.045512:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00000100:00000001:1.0:1788574921.621492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574921.621497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574921.621499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574921.621503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574921.621505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574921.621508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574921.621510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574921.621512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574921.621514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574922.645489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574922.645495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574922.645496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574922.645500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574922.645503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574922.645506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574922.645508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574922.645510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574922.645513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574923.669462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574923.669468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574923.669470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574923.669473:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574923.669477:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574923.669480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574923.669482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574923.669485:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574923.669487:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574924.693509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788574924.693517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788574924.693520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574924.693525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788574924.693529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788574924.693533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574924.693536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788574924.693539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788574924.693542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574925.206084:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574925.206093:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788574925.206532:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574925.206534:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788574925.206744:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574925.206746:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:0.0:1788574926.165468:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306529721, at 4306529728
00000100:00000040:1.0:1788574926.165468:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306529721, at 4306529728
00000100:00000010:1.0:1788574926.165482:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000100:00000010:0.0:1788574926.165484:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000100:00080000:0.0:1788574930.325494:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574930.325503:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574930.325505:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574930.325509:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574930.325513:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574930.325519:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11861
00000400:00000200:0.0:1788574930.325539:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574930.325547:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574930.325549:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574930.325552:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325554:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574930.325556:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325558:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574930.325561:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574930.325563:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574930.325565:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574930.325567:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574930.325569:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574930.325571:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574930.325573:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325575:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574930.325576:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325578:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574930.325580:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574930.325582:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325584:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574930.325586:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574930.325589:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574930.325591:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574930.325593:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574930.325594:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574930.325602:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788574930.325604:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574930.325607:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574930.325608:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574930.325610:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574930.325613:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574930.325616:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574930.325622:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574930.325628:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574930.325635:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393171328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574930.325643:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574930.325646:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574930.325650:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393171328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574930.325668:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574930.325672:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574930.325675:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574930.325677:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574930.325678:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574930.325682:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574930.325684:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11861
00000400:00000200:0.0:1788574930.325688:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574930.325691:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574930.325694:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574930.325697:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325699:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574930.325701:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325703:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574930.325706:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574930.325707:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574930.325709:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574930.325711:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574930.325713:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574930.325716:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574930.325718:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325721:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574930.325723:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325725:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574930.325729:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574930.325732:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325734:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574930.325736:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574930.325741:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574930.325744:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574930.325746:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574930.325750:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574930.325753:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574930.325755:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574930.325759:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788574930.325761:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574930.325763:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574930.325765:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574930.325767:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574930.325768:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574930.325770:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574930.325773:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574930.325776:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574930.325781:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393171456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574930.325789:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574930.325791:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574930.325795:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393171456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574930.325804:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574930.325808:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574930.325811:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574930.325813:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574930.325815:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574930.325818:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574930.325820:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11861
00000400:00000200:0.0:1788574930.325824:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574930.325844:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574930.325847:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574930.325850:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325852:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574930.325855:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325857:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574930.325860:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574930.325861:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574930.325863:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574930.325865:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574930.325868:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574930.325870:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574930.325873:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325875:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574930.325877:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325880:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574930.325882:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574930.325885:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574930.325887:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574930.325888:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574930.325892:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574930.325893:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574930.325896:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574930.325898:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574930.325900:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574930.325902:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574930.325906:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000800:00000200:2.1:1788574930.325906:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1788574930.325908:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.1:1788574930.325909:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
02000000:00000001:0.0:1788574930.325911:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574930.325913:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574930.325915:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574930.325916:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574930.325918:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574930.325921:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574930.325933:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574930.325937:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393171584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574930.325944:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574930.325947:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574930.325951:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393171584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574930.325959:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574930.325962:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574930.325966:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574930.325971:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11871)
00000800:00000200:2.1:1788574930.326263:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574930.326265:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788574930.326556:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574930.326558:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:0.0:1788574931.285496:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306534841, at 4306534848
00000100:00000040:1.0:1788574931.285496:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306534841, at 4306534848
00000100:00000010:1.0:1788574931.285511:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000100:00000010:0.0:1788574931.285513:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000100:00000001:0.0:1788574934.933579:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574934.933587:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574934.933589:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574934.933595:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574934.933599:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574934.933602:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574934.933606:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574934.933609:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574934.933612:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574935.445580:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574935.445596:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574935.445601:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574935.445606:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574935.445611:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574935.445618:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11866
00000400:00000200:1.0:1788574935.445628:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574935.445644:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574935.445648:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574935.445652:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445655:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574935.445658:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445661:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574935.445667:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574935.445669:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574935.445673:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574935.445677:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574935.445680:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574935.445683:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574935.445686:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445688:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574935.445689:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445691:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574935.445695:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574935.445705:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445708:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574935.445711:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574935.445714:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574935.445717:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574935.445719:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574935.445722:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574935.445733:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788574935.445736:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574935.445740:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574935.445741:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574935.445743:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574935.445747:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574935.445751:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574935.445760:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788574935.445767:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574935.445776:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393171712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574935.445785:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574935.445789:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574935.445793:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393171712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574935.445811:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574935.445813:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574935.445816:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574935.445818:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574935.445819:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574935.445821:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574935.445823:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11866
00000400:00000200:1.0:1788574935.445827:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574935.445829:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574935.445831:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574935.445833:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445835:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574935.445837:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445838:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574935.445840:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574935.445841:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574935.445842:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574935.445844:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574935.445846:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574935.445847:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574935.445850:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445852:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574935.445854:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445855:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574935.445857:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574935.445859:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574935.445861:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574935.445862:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574935.445868:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574935.445871:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574935.445874:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574935.445878:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574935.445882:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574935.445885:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574935.445888:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788574935.445891:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574935.445893:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574935.445895:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574935.445896:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574935.445899:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574935.445901:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574935.445907:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788574935.445911:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574935.445919:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393171840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574935.445928:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574935.445931:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574935.445934:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393171840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574935.445947:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574935.445954:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574935.445964:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574935.445967:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574935.445971:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574935.445975:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574935.445983:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11866
00000400:00000200:1.0:1788574935.445991:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574935.445996:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574935.445999:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574935.446005:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.446007:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574935.446011:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.446013:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574935.446015:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574935.446017:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574935.446019:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574935.446023:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574935.446026:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574935.446028:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574935.446032:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.446036:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574935.446037:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574935.446039:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574935.446040:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574935.446042:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574935.446044:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574935.446047:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574935.446051:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574935.446053:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574935.446054:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574935.446057:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574935.446061:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574935.446065:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574935.446069:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788574935.446072:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574935.446077:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574935.446078:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574935.446081:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574935.446083:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574935.446085:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574935.446089:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788574935.446094:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574935.446108:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393171968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574935.446118:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574935.446122:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574935.446128:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393171968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574935.446138:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574935.446141:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574935.446147:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574935.446155:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11876)
00000800:00000200:2.1:1788574935.446258:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574935.446262:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788574935.446784:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574935.446787:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788574935.447112:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574935.447115:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:0.0:1788574935.957479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574935.957500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574935.957504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574935.957512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574935.957517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574935.957531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574935.957535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574935.957540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574935.957544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574936.405510:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306539961, at 4306539968
00000100:00000040:1.0:1788574936.405528:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306539961, at 4306539968
00000100:00000010:0.0:1788574936.405530:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000100:00000010:1.0:1788574936.405544:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000100:00000001:0.0:1788574936.981558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574936.981566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574936.981568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574936.981574:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574936.981578:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574936.981582:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574936.981586:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574936.981589:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574936.981592:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574938.005484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574938.005492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574938.005494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574938.005500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574938.005504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574938.005508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574938.005510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574938.005513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574938.005516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574939.029535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574939.029543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574939.029545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574939.029550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574939.029554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574939.029558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574939.029560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574939.029563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574939.029566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574940.053486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574940.053493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574940.053495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574940.053500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574940.053503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574940.053507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574940.053509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574940.053511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574940.053512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574940.566037:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574940.566039:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1788574940.566312:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574940.566314:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1788574940.566537:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574940.566539:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:0.0:1788574941.077505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574941.077512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574941.077514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574941.077519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574941.077522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574941.077526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574941.077529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574941.077531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574941.077533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574941.525511:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306545081, at 4306545088
00000100:00000040:0.0:1788574941.525516:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306545081, at 4306545088
00000100:00000010:1.0:1788574941.525523:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:0.0:1788574941.525526:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:0.0:1788574942.101634:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574942.101640:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574942.101641:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574942.101646:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574942.101649:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574942.101652:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574942.101654:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574942.101656:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574942.101658:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574943.125546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574943.125553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574943.125556:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574943.125562:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574943.125566:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574943.125570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574943.125572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574943.125575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574943.125578:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788574943.637570:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788574943.637581:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788574943.637587:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788574943.637773:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574943.637794:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:1.0:1788574943.637795:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788574943.637799:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788574943.637803:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574944.149572:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574944.149582:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574944.149585:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574944.149590:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574944.149594:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574944.149599:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574944.149602:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574944.149605:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574944.149608:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574945.173480:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574945.173489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574945.173490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574945.173496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574945.173498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574945.173502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574945.173504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574945.173506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574945.173509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574945.685516:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574945.685527:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574945.685530:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574945.685535:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574945.685539:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574945.685546:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11876
00000400:00000200:0.0:1788574945.685569:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574945.685577:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574945.685581:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574945.685585:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685588:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574945.685591:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685594:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574945.685598:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574945.685601:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574945.685604:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574945.685607:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574945.685609:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574945.685612:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574945.685615:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685617:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574945.685618:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685620:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574945.685623:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574945.685626:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685629:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574945.685632:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574945.685635:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574945.685638:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574945.685640:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574945.685643:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574945.685652:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574945.685655:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574945.685659:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574945.685661:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574945.685663:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574945.685666:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574945.685669:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574945.685676:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574945.685683:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574945.685689:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393172480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574945.685696:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574945.685700:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574945.685703:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393172480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574945.685723:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574945.685725:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574945.685727:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574945.685728:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574945.685729:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574945.685731:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574945.685733:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11876
00000400:00000200:0.0:1788574945.685735:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574945.685737:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574945.685739:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574945.685741:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685742:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574945.685743:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685745:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574945.685746:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574945.685747:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574945.685748:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574945.685750:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574945.685751:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574945.685752:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574945.685754:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685755:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574945.685756:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685758:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574945.685759:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574945.685762:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685765:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574945.685768:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574945.685774:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574945.685776:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574945.685780:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574945.685784:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574945.685789:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574945.685792:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574945.685795:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788574945.685799:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574945.685802:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574945.685805:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574945.685809:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574945.685811:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574945.685814:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574945.685817:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574945.685822:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574945.685827:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393172608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574945.685836:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574945.685838:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574945.685843:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393172608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574945.685849:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574945.685854:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574945.685856:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574945.685857:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574945.685858:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574945.685861:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574945.685865:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11876
00000400:00000200:0.0:1788574945.685869:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574945.685873:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574945.685875:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574945.685879:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685880:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574945.685882:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685885:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574945.685889:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574945.685890:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574945.685892:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574945.685894:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574945.685897:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574945.685901:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574945.685906:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685909:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574945.685912:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685915:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574945.685918:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574945.685922:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574945.685924:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574945.685926:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574945.685928:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574945.685932:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574945.685933:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574945.685935:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574945.685939:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574945.685941:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574945.685944:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788574945.685947:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574945.685949:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574945.685951:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574945.685952:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574945.685953:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574945.685954:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574945.685959:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788574945.685962:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574945.685969:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393172736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574945.685978:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574945.685981:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574945.685986:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393172736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574945.686002:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574945.686005:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574945.686008:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574945.686014:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11886)
00000800:00000200:3.1:1788574945.686037:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574945.686040:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1788574945.686339:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574945.686341:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1788574945.686548:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574945.686550:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00000001:0.0:1788574946.197486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574946.197491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574946.197492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574946.197496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574946.197498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574946.197501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574946.197502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574946.197504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574946.197506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574946.645497:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306550201, at 4306550208
00000100:00000040:1.0:1788574946.645497:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306550201, at 4306550208
00000100:00000010:0.0:1788574946.645510:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:1.0:1788574946.645512:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000001:0.0:1788574947.221474:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574947.221482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574947.221485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574947.221491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574947.221495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574947.221499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574947.221502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574947.221504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574947.221507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574948.245517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574948.245523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574948.245524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574948.245529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574948.245531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574948.245534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574948.245537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574948.245538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574948.245541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574949.269531:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574949.269539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574949.269541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574949.269547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574949.269551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574949.269555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574949.269558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574949.269560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574949.269565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574950.293510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574950.293518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574950.293520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574950.293526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574950.293530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574950.293533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574950.293536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574950.293540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574950.293543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574950.805520:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574950.805529:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574950.805531:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574950.805534:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574950.805538:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574950.805544:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11881
00000400:00000200:0.0:1788574950.805552:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574950.805558:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574950.805560:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574950.805563:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805566:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574950.805568:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805569:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574950.805573:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574950.805574:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574950.805576:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574950.805578:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574950.805580:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574950.805582:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574950.805584:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805585:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574950.805586:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805588:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574950.805590:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574950.805593:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805595:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574950.805597:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574950.805599:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574950.805601:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574950.805603:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574950.805604:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574950.805612:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574950.805615:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574950.805618:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574950.805619:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574950.805621:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574950.805624:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574950.805627:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574950.805635:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788574950.805640:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574950.805647:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393172864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574950.805659:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574950.805663:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574950.805667:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393172864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574950.805685:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574950.805688:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574950.805690:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574950.805691:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574950.805692:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574950.805694:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574950.805696:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11881
00000400:00000200:0.0:1788574950.805698:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574950.805700:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574950.805702:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574950.805703:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805705:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574950.805710:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805713:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574950.805716:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574950.805718:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574950.805720:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574950.805723:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574950.805729:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574950.805733:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574950.805742:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805748:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574950.805753:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805758:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574950.805766:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574950.805772:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805791:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574950.805793:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574950.805799:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574950.805802:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574950.805805:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574950.805809:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574950.805815:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574950.805819:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574950.805824:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574950.805831:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574950.805839:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574950.805843:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574950.805847:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574950.805849:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574950.805853:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574950.805859:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788574950.805868:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574950.805884:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393172992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574950.805898:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574950.805902:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574950.805910:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393172992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574950.805928:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574950.805934:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574950.805943:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574950.805948:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574950.805953:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574950.805959:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574950.805966:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11881
00000400:00000200:0.0:1788574950.805980:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574950.805985:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574950.805987:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574950.805993:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.805997:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574950.806002:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.806005:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:3.1:1788574950.806007:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:0.0:1788574950.806008:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574950.806011:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:3.1:1788574950.806012:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1788574950.806014:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574950.806017:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574950.806021:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574950.806024:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574950.806029:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.806032:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574950.806036:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574950.806039:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574950.806043:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574950.806046:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574950.806049:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574950.806052:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574950.806055:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574950.806059:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574950.806061:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574950.806065:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574950.806069:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574950.806074:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574950.806078:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788574950.806083:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574950.806088:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574950.806091:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574950.806095:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574950.806098:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574950.806100:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574950.806105:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788574950.806110:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574950.806117:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393173120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574950.806127:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574950.806131:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574950.806135:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393173120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574950.806144:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574950.806148:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574950.806154:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574950.806164:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11892)
00000800:00000200:3.1:1788574950.806417:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574950.806419:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788574950.806654:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574950.806657:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1788574951.317543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574951.317550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574951.317552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574951.317557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574951.317561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574951.317565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574951.317568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574951.317571:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574951.317574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574951.765515:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306555321, at 4306555328
00000100:00000040:1.0:1788574951.765515:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306555321, at 4306555328
00000100:00000010:0.0:1788574951.765533:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:1.0:1788574951.765535:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:0.0:1788574952.341500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574952.341508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574952.341511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574952.341517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574952.341521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574952.341525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574952.341529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574952.341532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574952.341535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574953.365518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574953.365525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574953.365527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574953.365531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574953.365533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574953.365536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574953.365538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574953.365540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574953.365542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574954.389533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574954.389540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574954.389543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574954.389549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574954.389553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574954.389584:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574954.389588:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574954.389590:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574954.389594:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574955.413569:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574955.413576:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574955.413579:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574955.413584:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574955.413587:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574955.413592:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574955.413594:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574955.413596:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574955.413599:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574955.925527:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574955.925543:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574955.925547:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574955.925553:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574955.925559:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574955.925569:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11887
00000400:00000200:0.0:1788574955.925581:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574955.925589:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574955.925593:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574955.925596:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925600:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574955.925603:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925605:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574955.925610:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574955.925613:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574955.925616:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574955.925620:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574955.925623:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574955.925626:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574955.925629:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925631:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574955.925633:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925634:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574955.925637:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574955.925640:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925643:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574955.925646:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574955.925650:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574955.925653:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574955.925656:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574955.925658:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574955.925671:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788574955.925676:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574955.925680:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574955.925682:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574955.925686:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574955.925690:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574955.925695:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574955.925706:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788574955.925726:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574955.925739:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393173248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574955.925751:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574955.925756:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574955.925761:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393173248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574955.925788:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574955.925792:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574955.925796:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574955.925798:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574955.925799:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574955.925803:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574955.925807:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11887
00000400:00000200:0.0:1788574955.925812:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574955.925816:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574955.925819:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574955.925824:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925830:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574955.925837:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925842:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574955.925848:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574955.925851:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574955.925855:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574955.925861:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574955.925866:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574955.925872:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574955.925883:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925887:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574955.925894:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925898:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574955.925905:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574955.925913:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574955.925926:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574955.925930:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574955.925938:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574955.925945:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574955.925950:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574955.925957:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574955.925963:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574955.925968:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574955.925977:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788574955.925982:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574955.925986:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574955.925991:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574955.925995:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574955.926010:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574955.926012:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574955.926018:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788574955.926023:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574955.926031:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393173376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574955.926041:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574955.926046:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574955.926053:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393173376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574955.926072:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574955.926077:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574955.926084:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574955.926087:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574955.926089:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574955.926092:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574955.926096:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11887
00000400:00000200:0.0:1788574955.926114:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574955.926121:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574955.926126:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:3.1:1788574955.926137:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788574955.926140:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.1:1788574955.926142:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1788574955.926144:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574955.926149:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.926153:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574955.926157:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574955.926158:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574955.926160:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574955.926163:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574955.926168:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574955.926174:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574955.926180:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.926193:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574955.926200:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574955.926207:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574955.926212:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574955.926220:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574955.926225:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574955.926228:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574955.926233:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574955.926239:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574955.926244:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574955.926249:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574955.926252:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574955.926257:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574955.926265:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788574955.926272:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574955.926277:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574955.926282:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574955.926289:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574955.926292:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574955.926296:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574955.926305:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788574955.926313:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574955.926325:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393173504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574955.926347:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574955.926354:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574955.926365:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393173504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574955.926389:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574955.926426:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574955.926435:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574955.926443:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11897)
00000800:00000200:3.1:1788574955.926618:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574955.926619:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788574955.926841:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574955.926842:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1788574956.437525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574956.437531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574956.437533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574956.437538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574956.437541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574956.437544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574956.437547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574956.437549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574956.437552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574956.885504:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306560441, at 4306560448
00000100:00000040:1.0:1788574956.885504:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306560441, at 4306560448
00000100:00000010:1.0:1788574956.885520:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:0.0:1788574956.885525:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000001:0.0:1788574957.461504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574957.461511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574957.461513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574957.461519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574957.461522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574957.461526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574957.461528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574957.461531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574957.461534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574958.485556:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574958.485562:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574958.485564:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574958.485569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574958.485572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574958.485576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574958.485578:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574958.485580:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574958.485582:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574959.509510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574959.509516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574959.509518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574959.509523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574959.509527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574959.509530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574959.509533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574959.509535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574959.509537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574960.533533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574960.533539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574960.533541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574960.533546:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574960.533549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574960.533552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574960.533555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574960.533557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574960.533560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574961.045513:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574961.045525:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788574961.045528:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574961.045533:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574961.045538:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574961.045545:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11892
00000400:00000200:1.0:1788574961.045554:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574961.045560:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574961.045564:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574961.045568:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045571:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574961.045573:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045576:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574961.045580:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574961.045583:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788574961.045586:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574961.045588:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788574961.045590:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788574961.045592:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788574961.045594:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045596:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574961.045597:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045598:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574961.045600:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788574961.045603:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045605:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574961.045607:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574961.045610:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574961.045612:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574961.045614:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788574961.045616:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574961.045625:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788574961.045628:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788574961.045631:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574961.045632:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574961.045634:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574961.045636:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574961.045640:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574961.045646:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788574961.045651:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574961.045658:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393173632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574961.045666:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574961.045670:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574961.045673:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393173632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574961.045693:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574961.045695:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574961.045698:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788574961.045699:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574961.045700:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574961.045702:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574961.045704:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11892
00000400:00000200:1.0:1788574961.045707:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574961.045709:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574961.045710:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574961.045713:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045715:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574961.045717:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045719:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574961.045720:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574961.045722:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788574961.045723:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574961.045725:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788574961.045726:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788574961.045728:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788574961.045733:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045737:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574961.045741:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045743:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574961.045746:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788574961.045750:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045756:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574961.045759:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574961.045766:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574961.045772:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574961.045774:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574961.045780:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788574961.045785:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574961.045790:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574961.045793:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788574961.045797:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788574961.045800:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574961.045803:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574961.045805:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574961.045808:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574961.045810:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574961.045814:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788574961.045820:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574961.045829:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393173760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574961.045841:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574961.045844:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574961.045850:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393173760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574961.045856:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788574961.045858:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788574961.045860:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788574961.045861:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788574961.045862:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788574961.045864:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788574961.045866:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11892
00000400:00000200:1.0:1788574961.045869:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574961.045871:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788574961.045872:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788574961.045874:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045876:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574961.045877:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045879:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788574961.045880:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788574961.045881:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788574961.045882:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788574961.045883:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788574961.045884:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788574961.045886:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788574961.045887:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045889:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788574961.045890:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045891:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788574961.045892:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788574961.045894:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788574961.045895:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788574961.045896:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788574961.045898:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788574961.045899:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574961.045901:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788574961.045904:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788574961.045907:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574961.045908:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788574961.045911:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788574961.045915:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788574961.045919:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788574961.045922:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574961.045923:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788574961.045925:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788574961.045928:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788574961.045934:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788574961.045938:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788574961.045948:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393173888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788574961.045965:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788574961.045970:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574961.045974:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393173888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574961.045987:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574961.045991:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788574961.045999:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788574961.046003:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11902)
00000800:00000200:3.1:1788574961.046048:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574961.046050:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788574961.046344:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574961.046346:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788574961.046569:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574961.046571:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:0.0:1788574961.557543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574961.557550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574961.557553:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574961.557559:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574961.557564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574961.557568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574961.557571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574961.557574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574961.557577:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574962.005530:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306565561, at 4306565568
00000100:00000040:1.0:1788574962.005530:0:216131:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306565561, at 4306565568
00000100:00000010:0.0:1788574962.005547:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:1.0:1788574962.005550:0:216131:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000001:0.0:1788574962.581621:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574962.581628:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574962.581630:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574962.581637:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574962.581641:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574962.581658:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574962.581661:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574962.581664:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574962.581667:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574963.605607:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574963.605613:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574963.605615:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574963.605620:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574963.605623:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574963.605625:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574963.605627:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574963.605629:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574963.605631:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574964.629494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574964.629501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574964.629502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574964.629507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574964.629509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574964.629512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574964.629514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574964.629516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574964.629518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574965.653543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574965.653551:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574965.653554:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574965.653560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574965.653564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574965.653568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574965.653571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574965.653574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574965.653577:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574966.165515:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574966.166799:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574966.166802:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574966.166806:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574966.166810:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574966.166817:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11897
00000400:00000200:0.0:1788574966.166827:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574966.166834:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574966.166837:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574966.166841:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.166844:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574966.166847:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.166850:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574966.166854:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574966.166856:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574966.166859:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574966.166862:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574966.166865:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574966.166868:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574966.166871:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.166874:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574966.166876:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.166878:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574966.166881:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574966.166885:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574966.166888:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574966.166892:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574966.166896:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574966.166898:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574966.166901:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574966.166903:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574966.166912:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788574966.166916:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574966.166920:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574966.166921:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574966.166925:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574966.166928:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574966.166931:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574966.166938:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574966.166945:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574966.166956:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393174016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574966.166967:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574966.166971:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574966.166977:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393174016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574966.167000:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574966.167004:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574966.167008:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574966.167009:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574966.167011:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574966.167014:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574966.167033:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11897
00000400:00000200:0.0:1788574966.167040:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574966.167046:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574966.167050:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574966.167058:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167061:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574966.167066:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167071:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574966.167075:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574966.167078:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574966.167083:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574966.167086:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574966.167090:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574966.167094:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574966.167099:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167103:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574966.167108:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167111:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574966.167116:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574966.167121:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167127:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574966.167130:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574966.167139:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574966.167143:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574966.167148:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574966.167152:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574966.167157:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574966.167159:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574966.167167:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788574966.167171:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574966.167177:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574966.167180:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574966.167184:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574966.167185:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574966.167188:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574966.167193:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788574966.167208:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574966.167221:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393174144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574966.167241:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574966.167245:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574966.167250:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393174144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574966.167263:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574966.167267:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574966.167271:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574966.167274:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574966.167277:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574966.167279:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574966.167281:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11897
00000400:00000200:0.0:1788574966.167284:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574966.167286:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574966.167288:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574966.167290:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167291:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574966.167293:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167294:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574966.167295:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574966.167296:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574966.167297:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574966.167299:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574966.167300:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574966.167303:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574966.167307:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167312:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574966.167313:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167324:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574966.167329:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574966.167332:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574966.167334:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574966.167336:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574966.167339:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574966.167343:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574966.167346:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574966.167348:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574966.167351:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574966.167355:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574966.167359:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788574966.167362:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574966.167366:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574966.167368:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574966.167371:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574966.167373:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574966.167376:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574966.167380:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788574966.167384:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574966.167405:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393174272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574966.167412:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574966.167414:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574966.167417:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393174272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574966.167428:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574966.167434:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574966.167437:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574966.167456:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11907)
00000800:00000200:3.1:1788574966.167495:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574966.167498:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788574966.167941:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574966.167944:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788574966.168225:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574966.168228:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1788574966.677603:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574966.677620:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574966.677622:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574966.677628:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574966.677632:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574966.677636:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574966.677639:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574966.677643:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574966.677647:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574967.125521:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306570683, at 4306570688
00000100:00000040:1.0:1788574967.125530:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306570683, at 4306570688
00000100:00000010:0.0:1788574967.125546:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788574967.125551:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:0.0:1788574967.701564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574967.701570:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574967.701571:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574967.701583:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574967.701586:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574967.701589:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574967.701591:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574967.701593:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574967.701594:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574968.725502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574968.725508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574968.725510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574968.725514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574968.725517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574968.725520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574968.725522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574968.725524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574968.725527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574971.286029:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574971.286034:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574971.286468:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574971.286471:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788574971.286702:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574971.286704:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:0.0:1788574972.245523:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306575801, at 4306575808
00000100:00000040:1.0:1788574972.245523:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306575801, at 4306575808
00000100:00000010:1.0:1788574972.245540:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000100:00000010:0.0:1788574972.245544:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000800:00000200:2.1:1788574976.406087:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574976.406289:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574976.406479:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788574977.365480:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306580921, at 4306580928
00000100:00000010:1.0:1788574977.365499:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000100:00000040:0.0:1788574977.365548:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306580921, at 4306580928
00000100:00000010:0.0:1788574977.365562:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000100:00000001:0.0:1788574977.941544:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574977.941553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574977.941555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574977.941563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574977.941567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574977.941570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574977.941573:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574977.941576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574977.941579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574978.965500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574978.965506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574978.965508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574978.965512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574978.965515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574978.965518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574978.965520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574978.965522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574978.965524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788574979.477799:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788574979.477853:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788574979.477865:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574979.477871:0:2350:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788574979.477880:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788574979.477886:0:2350:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788574979.477893:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574979.477899:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:1.0:1788574979.477906:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574979.477912:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574979.477919:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65bb0
00000400:00000010:1.0:1788574979.477923:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65bb0.
00000100:00000001:1.0:1788574979.477929:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574979.477938:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0700 x1875444393174784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574979.477971:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574979.477976:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393174784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574979.477984:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574979.477986:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574979.477996:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574979.477999:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133c00.
00000400:00000200:1.0:1788574979.478004:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574979.478009:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574979.478015:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c655d8
00000400:00000010:1.0:1788574979.478017:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c655d8.
00000100:00000001:1.0:1788574979.478056:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574979.478067:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e1180 x1875444393174912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574979.478087:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574979.478095:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393174912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574979.478109:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574979.478114:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788574979.478132:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788574979.478151:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:1.0:1788574979.478172:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788574979.478184:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788574979.478190:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65440
00000400:00000010:1.0:1788574979.478193:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65440.
00000100:00000001:1.0:1788574979.478197:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788574979.478203:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2d80 x1875444393175040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574979.478236:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788574979.478248:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393175040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788574979.478281:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788574979.478297:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788574979.989480:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574979.989486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574979.989487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574979.989491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574979.989494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574979.989498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574979.989500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574979.989501:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574979.989503:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574981.013514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574981.013520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574981.013521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574981.013526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574981.013529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574981.013532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574981.013534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574981.013536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574981.013538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574981.525594:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574981.525602:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788574981.525605:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574981.525608:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574981.525612:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574981.525618:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11912
00000400:00000200:0.0:1788574981.525625:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574981.525630:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574981.525632:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574981.525635:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525637:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574981.525639:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525641:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574981.525644:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574981.525646:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788574981.525648:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574981.525650:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788574981.525652:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788574981.525653:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788574981.525655:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525656:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574981.525657:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525659:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574981.525661:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788574981.525664:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525666:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574981.525668:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574981.525671:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574981.525673:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574981.525675:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788574981.525676:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574981.525681:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788574981.525684:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788574981.525687:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574981.525688:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574981.525691:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574981.525693:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574981.525697:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574981.525705:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788574981.525712:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574981.525722:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393175168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574981.525733:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574981.525738:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574981.525743:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393175168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574981.525771:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574981.525775:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574981.525779:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788574981.525780:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574981.525782:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574981.525786:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574981.525789:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11912
00000400:00000200:0.0:1788574981.525793:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574981.525796:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574981.525799:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574981.525802:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525804:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574981.525812:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525815:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574981.525818:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574981.525819:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788574981.525823:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574981.525827:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788574981.525830:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788574981.525832:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788574981.525838:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525840:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574981.525843:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525845:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574981.525848:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788574981.525851:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574981.525854:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574981.525856:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574981.525863:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574981.525870:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574981.525873:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574981.525881:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788574981.525887:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574981.525889:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574981.525899:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788574981.525906:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788574981.525915:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574981.525919:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574981.525921:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574981.525922:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574981.525925:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574981.525929:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788574981.525937:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574981.525952:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393175296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574981.525964:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574981.525971:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574981.525984:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393175296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574981.526014:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788574981.526020:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788574981.526034:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788574981.526038:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788574981.526040:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788574981.526045:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788574981.526053:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11912
00000400:00000200:0.0:1788574981.526067:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788574981.526074:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788574981.526079:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788574981.526086:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.526096:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574981.526099:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.526103:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788574981.526126:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788574981.526130:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788574981.526132:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788574981.526134:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788574981.526139:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788574981.526147:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788574981.526158:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.526165:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788574981.526172:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788574981.526176:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788574981.526184:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788574981.526197:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788574981.526206:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788574981.526211:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788574981.526218:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788574981.526224:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574981.526232:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788574981.526236:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788574981.526248:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574981.526250:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788574981.526257:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788574981.526264:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788574981.526269:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788574981.526271:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574981.526273:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788574981.526278:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788574981.526284:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788574981.526290:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788574981.526305:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574981.526318:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393175424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788574981.526345:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788574981.526352:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574981.526368:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393175424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788574981.526381:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574981.526449:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788574981.526455:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788574981.526463:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11922)
00000800:00000200:2.1:1788574981.526510:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574981.526515:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788574981.527054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574981.527057:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788574981.527470:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574981.527474:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1788574982.037528:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574982.037537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574982.037540:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574982.037546:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574982.037549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574982.037553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574982.037556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574982.037558:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574982.037561:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574982.485523:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306586042, at 4306586048
00000100:00000040:0.0:1788574982.485530:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306586042, at 4306586048
00000100:00000010:1.0:1788574982.485536:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00000100:00000010:0.0:1788574982.485545:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000800:00000200:1.0:1788574982.549527:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788574982.549533:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788574982.549537:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788574982.549770:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788574982.549782:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788574982.549791:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788574982.549797:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788574982.549800:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574983.061534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574983.061543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574983.061546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574983.061551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574983.061555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574983.061559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574983.061562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574983.061564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574983.061567:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574984.085475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574984.085483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574984.085486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574984.085499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574984.085504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574984.085508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574984.085511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574984.085515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574984.085518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574985.109534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574985.109544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574985.109546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574985.109553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574985.109557:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574985.109561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574985.109564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574985.109567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574985.109570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574986.133487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574986.133493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574986.133495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574986.133499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574986.133502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574986.133505:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574986.133507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574986.133510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574986.133512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574986.646292:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574986.646298:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788574986.646959:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574986.646961:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788574986.647203:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574986.647205:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:0.0:1788574987.157603:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574987.157614:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574987.157617:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574987.157624:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574987.157630:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574987.157635:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574987.157638:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574987.157641:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574987.157645:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788574987.605539:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306591162, at 4306591168
00000100:00000040:0.0:1788574987.605540:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306591162, at 4306591168
00000100:00000010:0.0:1788574987.605558:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:1.0:1788574987.605564:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:0.0:1788574988.181564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574988.181574:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574988.181576:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574988.181582:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574988.181585:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574988.181589:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574988.181591:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574988.181594:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574988.181597:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574989.205583:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574989.205591:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574989.205594:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574989.205600:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574989.205603:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574989.205608:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574989.205611:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574989.205613:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574989.205616:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574990.229559:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574990.229566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574990.229568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574990.229571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574990.229574:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574990.229578:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574990.229580:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574990.229582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574990.229584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574991.253488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574991.253495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574991.253497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574991.253501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574991.253503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574991.253506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574991.253508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574991.253511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574991.253513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788574991.766025:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574991.766030:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1788574991.766371:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574991.766373:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1788574991.766606:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788574991.766609:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:0.0:1788574992.277489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574992.277495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574992.277497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574992.277502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574992.277506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574992.277509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574992.277511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574992.277513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574992.277515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574992.725462:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306596281, at 4306596288
00000100:00000010:0.0:1788574992.725476:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000040:1.0:1788574992.725490:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306596281, at 4306596288
00000100:00000010:1.0:1788574992.725500:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000001:0.0:1788574993.301500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574993.301508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574993.301511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574993.301515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574993.301518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574993.301521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574993.301523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574993.301525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574993.301527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574994.325479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574994.325486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574994.325488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574994.325493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574994.325496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574994.325499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574994.325501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574994.325503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574994.325506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574995.349467:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574995.349475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574995.349477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574995.349482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574995.349486:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574995.349490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574995.349493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574995.349496:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574995.349499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574996.373501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574996.373508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574996.373509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574996.373514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574996.373517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574996.373520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574996.373522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574996.373524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574996.373526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788574996.886305:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574996.886309:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788574996.886718:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574996.886720:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788574996.887027:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788574996.887028:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:0.0:1788574997.397493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574997.397502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574997.397504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574997.397511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574997.397515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574997.397518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574997.397521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574997.397524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574997.397528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788574997.845499:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306601401, at 4306601408
00000100:00000010:0.0:1788574997.845513:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00000100:00000040:1.0:1788574997.845515:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306601402, at 4306601408
00000100:00000010:1.0:1788574997.845525:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000100:00000001:0.0:1788574998.421484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574998.421490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574998.421491:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574998.421495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574998.421498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574998.421501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574998.421503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574998.421505:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574998.421507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788574999.445501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788574999.445510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788574999.445513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788574999.445518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788574999.445521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788574999.445524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574999.445526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788574999.445528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788574999.445530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575000.469524:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575000.469530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575000.469532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575000.469537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575000.469541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575000.469544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575000.469547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575000.469550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575000.469554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575001.493558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575001.493565:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575001.493567:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575001.493574:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575001.493578:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575001.493584:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575001.493587:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575001.493589:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575001.493593:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575002.005541:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575002.005553:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575002.005556:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575002.005560:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575002.005565:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575002.005573:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11933
00000400:00000200:0.0:1788575002.005582:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575002.005589:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575002.005592:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575002.005596:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005599:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575002.005601:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005604:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575002.005608:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575002.005610:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575002.005613:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575002.005616:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575002.005618:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575002.005621:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575002.005624:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005626:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575002.005628:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005630:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575002.005633:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575002.005637:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005639:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575002.005642:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575002.005645:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575002.005648:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575002.005650:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575002.005652:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575002.005661:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575002.005665:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575002.005669:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575002.005670:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575002.005673:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575002.005676:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575002.005680:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575002.005688:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788575002.005695:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575002.005704:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393176704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575002.005715:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575002.005719:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575002.005723:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393176704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575002.005745:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575002.005748:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575002.005752:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575002.005753:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575002.005755:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575002.005757:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575002.005760:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11933
00000400:00000200:0.0:1788575002.005764:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575002.005767:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575002.005770:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575002.005773:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005776:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575002.005779:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005781:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575002.005784:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575002.005785:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575002.005787:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575002.005789:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575002.005792:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575002.005797:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575002.005802:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005805:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575002.005809:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005813:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575002.005816:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575002.005824:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005830:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575002.005834:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575002.005842:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575002.005846:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575002.005849:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575002.005855:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575002.005863:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575002.005868:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575002.005871:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788575002.005872:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575002.005878:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575002.005880:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575002.005882:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575002.005884:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575002.005887:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575002.005892:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788575002.005897:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575002.005905:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393176832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575002.005913:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575002.005916:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575002.005923:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393176832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575002.005933:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575002.005936:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575002.005941:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575002.005942:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575002.005944:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575002.005949:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575002.005951:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11933
00000400:00000200:0.0:1788575002.005955:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575002.005961:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575002.005963:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575002.005966:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005969:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575002.005973:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.005975:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575002.005979:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575002.005981:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575002.005984:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575002.005986:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575002.005989:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575002.005990:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575002.006010:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.006015:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575002.006018:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575002.006023:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575002.006027:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575002.006031:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575002.006035:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575002.006038:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575002.006041:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575002.006044:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575002.006048:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575002.006050:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575002.006052:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575002.006057:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575002.006061:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788575002.006066:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575002.006068:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575002.006069:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575002.006070:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575002.006071:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575002.006075:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575002.006079:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788575002.006083:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575002.006089:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393176960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575002.006098:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575002.006100:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575002.006105:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393176960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575002.006116:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575002.006122:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575002.006127:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575002.006134:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11943)
00000800:00000200:2.1:1788575002.006206:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575002.006210:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788575002.006709:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575002.006712:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788575002.007071:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575002.007074:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:0.0:1788575002.517536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575002.517542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575002.517543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575002.517547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575002.517550:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575002.517553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575002.517556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575002.517558:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575002.517560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575002.965532:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306606521, at 4306606528
00000100:00000040:0.0:1788575002.965533:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306606521, at 4306606528
00000100:00000010:0.0:1788575002.965548:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000100:00000010:1.0:1788575002.965550:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000100:00000001:0.0:1788575003.541498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575003.541505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575003.541507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575003.541511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575003.541514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575003.541517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575003.541519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575003.541521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575003.541523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575007.126309:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575007.126312:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575007.126751:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575007.126753:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575007.127006:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575007.127008:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000040:0.0:1788575008.085555:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306611641, at 4306611648
00000100:00000040:1.0:1788575008.085555:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306611642, at 4306611648
00000100:00000010:0.0:1788575008.085581:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000100:00000010:1.0:1788575008.085587:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00000800:00000200:2.1:1788575012.246368:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575012.246373:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575012.246756:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575012.246758:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575012.247003:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575012.247006:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:1.0:1788575013.205519:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306616762, at 4306616768
00000100:00000040:0.0:1788575013.205530:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306616762, at 4306616768
00000100:00000010:1.0:1788575013.205537:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000100:00000010:0.0:1788575013.205548:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000100:00080000:0.0:1788575017.365536:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575017.365548:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575017.365552:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575017.365557:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575017.365562:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575017.365571:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11948
00000400:00000200:0.0:1788575017.365581:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575017.365587:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575017.365590:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575017.365593:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365596:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575017.365598:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365607:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575017.365611:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575017.365612:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575017.365615:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575017.365618:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575017.365620:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575017.365622:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575017.365625:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365627:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575017.365629:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365630:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575017.365633:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575017.365637:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365638:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575017.365641:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575017.365644:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575017.365646:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575017.365648:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575017.365650:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575017.365661:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788575017.365664:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575017.365668:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575017.365669:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575017.365672:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575017.365675:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575017.365680:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575017.365688:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788575017.365696:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575017.365706:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393177856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575017.365716:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575017.365720:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575017.365724:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393177856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575017.365739:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575017.365742:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575017.365745:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575017.365747:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575017.365748:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575017.365751:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575017.365757:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11948
00000400:00000200:0.0:1788575017.365769:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575017.365774:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575017.365779:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575017.365785:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365787:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575017.365791:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365795:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575017.365799:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575017.365802:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575017.365805:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575017.365809:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575017.365812:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575017.365816:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575017.365821:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365823:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575017.365825:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365829:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575017.365831:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575017.365834:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365837:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575017.365838:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575017.365844:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575017.365847:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575017.365849:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575017.365858:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575017.365862:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575017.365863:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575017.365867:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788575017.365870:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575017.365872:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575017.365874:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575017.365878:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575017.365881:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575017.365884:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575017.365890:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788575017.365896:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575017.365904:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393177984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575017.365915:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575017.365917:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575017.365924:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393177984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575017.365937:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575017.365939:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575017.365947:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575017.365952:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575017.365955:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575017.365957:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575017.365961:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11948
00000400:00000200:0.0:1788575017.365966:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575017.365972:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575017.365974:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575017.365979:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365984:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575017.365989:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.365993:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575017.365997:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575017.365999:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575017.366001:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575017.366004:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575017.366006:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575017.366009:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575017.366013:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.366017:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575017.366021:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575017.366026:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575017.366028:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575017.366033:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575017.366037:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575017.366038:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575017.366041:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575017.366045:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575017.366046:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575017.366048:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575017.366052:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575017.366054:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575017.366059:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788575017.366064:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575017.366068:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:2.1:1788575017.366069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1788575017.366070:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575017.366075:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:2.1:1788575017.366075:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
02000000:00000001:0.0:1788575017.366076:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575017.366079:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575017.366084:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788575017.366088:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575017.366095:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393178112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575017.366100:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575017.366102:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575017.366104:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393178112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575017.366110:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575017.366113:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575017.366117:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575017.366125:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11958)
00000800:00000200:2.1:1788575017.366564:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575017.366566:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575017.366788:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575017.366789:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:1.0:1788575018.325550:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306621881, at 4306621888
00000100:00000010:1.0:1788575018.325568:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00000100:00000040:0.0:1788575018.325681:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306621881, at 4306621888
00000100:00000010:0.0:1788575018.325689:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000800:00000200:1.0:1788575020.437544:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575020.437553:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788575020.437558:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575020.437727:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575020.437729:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575020.437744:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575020.437751:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575020.437754:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788575022.486171:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575022.486175:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575022.486644:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575022.486646:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575022.486932:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575022.486935:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:1.0:1788575023.445564:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306627001, at 4306627008
00000100:00000040:0.0:1788575023.445565:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306627001, at 4306627008
00000100:00000010:1.0:1788575023.445607:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000100:00000010:0.0:1788575023.445611:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000800:00000200:2.1:1788575027.606259:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575027.606268:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575027.606814:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575027.606817:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575027.607125:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575027.607128:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:0.0:1788575028.565531:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306632121, at 4306632128
00000100:00000040:1.0:1788575028.565531:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306632122, at 4306632128
00000100:00000010:1.0:1788575028.565550:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000100:00000010:0.0:1788575028.565553:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00000100:00080000:0.0:1788575032.725529:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575032.725546:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575032.725549:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575032.725553:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575032.725557:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575032.725564:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11963
00000400:00000200:0.0:1788575032.725572:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575032.725578:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575032.725581:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575032.725584:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725587:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575032.725590:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725592:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575032.725596:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575032.725598:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575032.725601:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575032.725604:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575032.725606:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575032.725609:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575032.725613:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725615:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575032.725617:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725619:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575032.725622:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575032.725625:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725627:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575032.725630:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575032.725633:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575032.725635:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575032.725637:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575032.725640:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575032.725649:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788575032.725653:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575032.725657:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575032.725659:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575032.725662:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575032.725665:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575032.725669:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575032.725676:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788575032.725682:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575032.725691:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393179008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575032.725702:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575032.725707:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575032.725712:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393179008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575032.725736:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575032.725740:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575032.725744:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575032.725745:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575032.725748:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575032.725750:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575032.725753:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11963
00000400:00000200:0.0:1788575032.725757:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575032.725760:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575032.725763:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575032.725767:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725770:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575032.725772:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725775:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575032.725778:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575032.725782:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575032.725787:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575032.725791:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575032.725795:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575032.725800:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575032.725821:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725826:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575032.725831:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725837:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575032.725840:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575032.725846:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575032.725852:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575032.725855:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575032.725862:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575032.725867:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575032.725870:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575032.725875:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575032.725881:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575032.725883:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575032.725934:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575032.725939:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575032.725943:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575032.725946:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575032.725951:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575032.725954:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575032.725958:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575032.725965:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575032.725972:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575032.725982:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393179136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575032.725995:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575032.726001:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575032.726008:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393179136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575032.726025:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575032.726030:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575032.726038:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575032.726043:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575032.726047:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575032.726054:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575032.726061:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11963
00000400:00000200:0.0:1788575032.726077:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575032.726086:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575032.726089:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575032.726097:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.726102:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788575032.726106:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788575032.726109:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575032.726110:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1788575032.726115:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575032.726120:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575032.726125:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575032.726130:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575032.726134:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575032.726139:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575032.726144:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575032.726150:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.726155:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575032.726161:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575032.726165:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575032.726169:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575032.726175:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575032.726181:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575032.726186:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575032.726190:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575032.726194:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575032.726197:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575032.726201:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575032.726206:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575032.726210:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575032.726216:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788575032.726222:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575032.726229:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575032.726232:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575032.726235:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575032.726238:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575032.726241:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575032.726247:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575032.726252:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575032.726262:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393179264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575032.726279:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575032.726283:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575032.726294:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393179264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575032.726321:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575032.726326:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575032.726335:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575032.726345:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11973)
00000800:00000200:2.1:1788575032.726592:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575032.726595:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575032.726845:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575032.726848:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1788575033.685521:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306637241, at 4306637248
00000100:00000010:0.0:1788575033.685535:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000100:00000040:1.0:1788575033.685537:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306637241, at 4306637248
00000100:00000010:1.0:1788575033.685550:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000100:00000001:0.0:1788575035.285574:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575035.285583:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575035.285586:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575035.285591:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575035.285595:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575035.285599:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575035.285601:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575035.285604:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575035.285606:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575036.309521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575036.309530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575036.309532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575036.309537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575036.309540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575036.309545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575036.309547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575036.309550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575036.309552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575037.333533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575037.333541:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575037.333543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575037.333549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575037.333552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575037.333555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575037.333558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575037.333561:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575037.333564:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575037.846107:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575037.846119:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575037.846578:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575037.846581:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575037.846861:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575037.846863:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:0.0:1788575038.357476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575038.357483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575038.357485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575038.357490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575038.357494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575038.357497:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575038.357500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575038.357502:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575038.357505:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575038.805466:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306642361, at 4306642368
00000100:00000010:0.0:1788575038.805477:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000100:00000040:1.0:1788575038.805502:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306642361, at 4306642368
00000100:00000010:1.0:1788575038.805514:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00000100:00000001:0.0:1788575039.381521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575039.381526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575039.381527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575039.381531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575039.381534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575039.381537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575039.381539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575039.381540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575039.381542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575040.405460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575040.405466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575040.405468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575040.405472:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575040.405475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575040.405477:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575040.405479:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575040.405481:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575040.405482:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575041.429537:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575041.429545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575041.429547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575041.429552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575041.429556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575041.429561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575041.429563:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575041.429566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575041.429568:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575042.965494:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575042.965504:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575042.965507:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575042.965510:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575042.965514:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575042.965521:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11974
00000400:00000200:0.0:1788575042.965529:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575042.965535:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575042.965538:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575042.965541:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965544:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575042.965547:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965549:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575042.965553:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575042.965555:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575042.965558:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575042.965560:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575042.965563:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575042.965565:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575042.965568:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965570:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575042.965572:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965573:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575042.965576:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575042.965580:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965582:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575042.965585:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575042.965588:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575042.965591:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575042.965593:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575042.965596:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575042.965605:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788575042.965609:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575042.965612:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575042.965614:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575042.965617:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575042.965620:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575042.965623:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575042.965630:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788575042.965636:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575042.965645:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393179776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575042.965655:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575042.965658:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575042.965663:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393179776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575042.965683:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575042.965686:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575042.965690:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575042.965691:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575042.965693:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575042.965695:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575042.965698:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11974
00000400:00000200:0.0:1788575042.965702:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575042.965705:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575042.965708:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575042.965711:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965714:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575042.965716:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965718:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575042.965722:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575042.965723:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575042.965725:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575042.965728:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575042.965730:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575042.965733:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575042.965736:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965738:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575042.965740:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965742:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575042.965745:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575042.965749:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965754:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575042.965759:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575042.965769:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575042.965773:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575042.965775:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575042.965780:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575042.965785:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575042.965792:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575042.965801:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788575042.965807:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575042.965817:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575042.965822:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575042.965829:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575042.965835:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575042.965842:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575042.965850:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788575042.965862:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575042.965875:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393179904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575042.965894:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575042.965900:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575042.965910:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393179904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575042.965937:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575042.965944:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575042.965953:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575042.965957:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575042.965961:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575042.965969:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575042.965976:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11974
00000400:00000200:0.0:1788575042.965982:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575042.965985:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575042.965987:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575042.965991:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.965999:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575042.966004:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.966007:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575042.966015:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575042.966019:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575042.966021:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575042.966027:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575042.966035:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575042.966055:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575042.966061:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.966069:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575042.966077:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575042.966086:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575042.966093:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575042.966098:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575042.966105:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575042.966112:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575042.966123:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575042.966128:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575042.966136:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575042.966141:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575042.966147:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575042.966150:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575042.966157:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575042.966163:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575042.966169:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575042.966172:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575042.966173:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575042.966177:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575042.966180:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575042.966183:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575042.966187:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788575042.966191:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575042.966198:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393180032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575042.966218:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575042.966227:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575042.966236:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393180032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575042.966254:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575042.966258:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575042.966269:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575042.966282:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11984)
00000800:00000200:2.1:1788575042.966513:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575042.966665:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788575043.925517:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306647481, at 4306647488
00000100:00000040:1.0:1788575043.925517:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306647481, at 4306647488
00000100:00000010:1.0:1788575043.925536:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000100:00000010:0.0:1788575043.925540:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00000800:00000200:3.1:1788575048.086839:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575048.086843:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575048.087190:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575048.087192:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575048.087489:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575048.087491:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:0.0:1788575049.045490:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306652602, at 4306652608
00000100:00000040:1.0:1788575049.045490:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306652602, at 4306652608
00000100:00000010:0.0:1788575049.045508:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:1.0:1788575049.045511:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00080000:0.0:1788575053.205527:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575053.205539:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575053.205542:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575053.205546:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575053.205550:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575053.205557:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11984
00000400:00000200:0.0:1788575053.205566:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575053.205572:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575053.205575:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575053.205577:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205580:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575053.205582:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205584:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575053.205587:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575053.205589:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575053.205591:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575053.205594:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575053.205595:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575053.205597:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575053.205598:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205599:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575053.205601:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205602:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575053.205605:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575053.205607:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205609:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575053.205612:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575053.205615:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575053.205617:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575053.205620:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575053.205622:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575053.205633:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788575053.205636:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575053.205640:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575053.205641:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575053.205644:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575053.205647:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575053.205652:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575053.205661:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788575053.205668:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575053.205676:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393180544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575053.205685:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575053.205689:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575053.205692:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393180544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575053.205712:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575053.205714:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575053.205717:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575053.205718:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575053.205720:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575053.205722:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575053.205727:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11984
00000400:00000200:0.0:1788575053.205730:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575053.205731:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575053.205734:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575053.205735:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205738:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575053.205739:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205741:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575053.205745:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575053.205746:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575053.205748:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575053.205752:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575053.205754:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575053.205756:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575053.205760:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205762:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575053.205765:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205767:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575053.205771:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575053.205779:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575053.205785:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575053.205789:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575053.205798:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575053.205808:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575053.205819:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575053.205828:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575053.205837:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575053.205853:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575053.205863:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788575053.205877:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575053.205889:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575053.205893:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575053.205900:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575053.205902:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575053.205904:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575053.205908:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788575053.205911:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575053.205916:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393180672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575053.205938:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575053.205943:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575053.205949:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393180672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575053.205961:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575053.205964:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575053.205979:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575053.205981:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575053.205986:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575053.205996:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575053.206002:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11984
00000400:00000200:0.0:1788575053.206007:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575053.206019:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575053.206023:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575053.206029:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.206034:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575053.206039:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.206042:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575053.206048:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575053.206049:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575053.206051:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575053.206054:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575053.206061:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575053.206069:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575053.206079:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.206085:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575053.206090:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575053.206099:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575053.206107:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575053.206116:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575053.206123:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575053.206126:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575053.206133:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575053.206144:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575053.206155:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575053.206160:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575053.206166:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575053.206172:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575053.206180:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788575053.206186:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575053.206191:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575053.206194:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575053.206199:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575053.206203:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575053.206207:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575053.206212:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788575053.206222:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575053.206233:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393180800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575053.206256:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575053.206261:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575053.206267:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393180800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1788575053.206288:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575053.206296:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:0.0:1788575053.206302:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575053.206309:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575053.206316:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575053.206327:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11994)
00000800:00000200:3.1:1788575053.206671:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575053.206673:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575053.206860:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575053.206863:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:0.0:1788575054.165502:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306657721, at 4306657728
00000100:00000040:1.0:1788575054.165502:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306657722, at 4306657728
00000100:00000010:1.0:1788575054.165517:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:0.0:1788575054.165519:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:0.0:1788575056.789535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575056.789543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575056.789547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575056.789554:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575056.789559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575056.789563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575056.789566:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575056.789570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575056.789574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575057.813543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575057.813549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575057.813550:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575057.813555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575057.813558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575057.813561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575057.813564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575057.813565:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575057.813568:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788575058.325545:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575058.325558:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788575058.325568:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575058.325718:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575058.325739:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:0.0:1788575058.325741:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575058.325749:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575058.325754:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788575058.326099:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575058.326103:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575058.326469:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575058.326471:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575058.326677:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575058.326679:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1788575058.837570:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575058.837578:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575058.837580:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575058.837585:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575058.837589:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575058.837593:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575058.837595:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575058.837598:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575058.837601:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575059.285546:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306662841, at 4306662848
00000100:00000040:1.0:1788575059.285546:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306662841, at 4306662848
00000100:00000010:0.0:1788575059.285566:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000100:00000010:1.0:1788575059.285568:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000100:00000001:0.0:1788575059.861517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575059.861528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575059.861531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575059.861538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575059.861543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575059.861547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575059.861550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575059.861554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575059.861557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575060.885591:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575060.885600:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575060.885602:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575060.885608:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575060.885616:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575060.885620:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575060.885623:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575060.885626:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575060.885628:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575061.909512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575061.909518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575061.909520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575061.909524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575061.909527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575061.909530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575061.909533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575061.909535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575061.909536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575062.933531:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575062.933539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575062.933542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575062.933548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575062.933552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575062.933556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575062.933559:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575062.933562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575062.933565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575063.446070:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575063.446075:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788575063.446429:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575063.446431:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788575063.446681:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575063.446683:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:0.0:1788575063.957521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575063.957540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575063.957542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575063.957548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575063.957552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575063.957555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575063.957558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575063.957560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575063.957563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575064.405520:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306667961, at 4306667968
00000100:00000040:1.0:1788575064.405520:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306667961, at 4306667968
00000100:00000010:1.0:1788575064.405552:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000100:00000010:0.0:1788575064.405556:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00000100:00000001:0.0:1788575064.981539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575064.981570:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575064.981572:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575064.981581:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575064.981584:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575064.981589:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575064.981591:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575064.981594:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575064.981597:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575066.005543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575066.005554:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575066.005557:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575066.005565:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575066.005594:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575066.005599:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575066.005601:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575066.005604:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575066.005607:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575067.029534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575067.029542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575067.029544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575067.029549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575067.029552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575067.029558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575067.029560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575067.029563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575067.029566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575068.053538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575068.053548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575068.053551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575068.053559:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575068.053564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575068.053568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575068.053571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575068.053575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575068.053579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575068.565517:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575068.565528:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575068.565531:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575068.565536:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575068.565541:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575068.565548:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 11999
00000400:00000200:0.0:1788575068.565558:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575068.565564:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575068.565567:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575068.565570:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565573:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575068.565575:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565577:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575068.565581:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575068.565583:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575068.565585:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575068.565588:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575068.565590:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575068.565591:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575068.565593:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565594:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575068.565596:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565597:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575068.565601:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575068.565605:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565609:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575068.565612:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575068.565616:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575068.565620:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575068.565623:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575068.565626:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575068.565650:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788575068.565654:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575068.565661:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575068.565662:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575068.565667:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575068.565670:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575068.565676:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575068.565686:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575068.565695:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575068.565709:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393181696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575068.565721:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575068.565726:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575068.565731:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393181696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575068.565750:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575068.565753:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575068.565756:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575068.565757:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575068.565759:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575068.565761:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575068.565764:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11999
00000400:00000200:0.0:1788575068.565768:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575068.565771:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575068.565773:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575068.565776:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565786:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575068.565790:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565793:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575068.565795:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575068.565797:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575068.565799:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575068.565802:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575068.565808:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575068.565811:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575068.565817:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565820:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575068.565826:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565831:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575068.565836:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575068.565843:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575068.565848:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575068.565852:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575068.565861:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575068.565869:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575068.565874:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575068.565879:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575068.565883:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575068.565886:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575068.565894:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788575068.565897:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575068.565901:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575068.565903:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575068.565907:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575068.565909:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575068.565912:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575068.565919:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575068.565924:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575068.565934:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393181824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575068.565950:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575068.565955:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575068.565964:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393181824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575068.566001:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575068.566007:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575068.566010:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575068.566013:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575068.566016:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575068.566018:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575068.566024:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 11999
00000400:00000200:0.0:1788575068.566032:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575068.566036:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575068.566040:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575068.566048:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.566051:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575068.566057:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.566062:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575068.566065:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575068.566068:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575068.566071:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575068.566076:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575068.566080:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575068.566085:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575068.566092:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.566097:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575068.566099:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575068.566101:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575068.566105:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575068.566111:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575068.566115:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575068.566119:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575068.566122:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575068.566125:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575068.566127:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575068.566130:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575068.566134:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575068.566138:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575068.566148:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788575068.566154:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575068.566161:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575068.566162:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575068.566165:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575068.566168:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575068.566172:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575068.566179:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788575068.566186:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575068.566197:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393181952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1788575068.566198:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575068.566205:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:0.0:1788575068.566216:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575068.566222:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575068.566233:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393181952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575068.566252:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575068.566256:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575068.566262:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575068.566269:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12009)
00000800:00000200:2.1:1788575068.566606:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575068.566608:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788575068.566881:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575068.566883:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:0.0:1788575069.077536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575069.077545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575069.077547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575069.077552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575069.077556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575069.077560:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575069.077563:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575069.077566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575069.077570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575069.525562:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306673081, at 4306673088
00000100:00000040:1.0:1788575069.525562:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306673081, at 4306673088
00000100:00000010:0.0:1788575069.525584:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000100:00000010:1.0:1788575069.525588:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000100:00000001:0.0:1788575070.101557:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575070.101567:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575070.101569:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575070.101575:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575070.101579:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575070.101585:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575070.101588:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575070.101591:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575070.101593:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575071.125581:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575071.125591:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575071.125595:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575071.125605:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575071.125613:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575071.125620:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575071.125625:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575071.125628:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575071.125633:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575072.149481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575072.149487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575072.149488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575072.149492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575072.149495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575072.149498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575072.149500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575072.149502:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575072.149504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575073.173539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575073.173548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575073.173550:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575073.173556:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575073.173559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575073.173565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575073.173567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575073.173569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575073.173572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575073.686152:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575073.686157:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788575073.686645:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575073.686647:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788575073.686895:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575073.686897:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:0.0:1788575074.197512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575074.197519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575074.197527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575074.197533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575074.197536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575074.197539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575074.197541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575074.197543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575074.197546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575074.645539:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306678201, at 4306678208
00000100:00000040:1.0:1788575074.645539:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306678201, at 4306678208
00000100:00000010:1.0:1788575074.645557:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00000100:00000010:0.0:1788575074.645559:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000100:00000001:0.0:1788575075.221485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575075.221491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575075.221493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575075.221498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575075.221501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575075.221504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575075.221506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575075.221507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575075.221510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575076.245512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575076.245521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575076.245523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575076.245527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575076.245530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575076.245534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575076.245536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575076.245538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575076.245541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575077.269487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575077.269493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575077.269495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575077.269499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575077.269501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575077.269505:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575077.269507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575077.269509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575077.269511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575078.293500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575078.293506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575078.293508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575078.293512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575078.293516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575078.293519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575078.293521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575078.293523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575078.293525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575078.805637:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575078.805646:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575078.805648:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575078.805651:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575078.805654:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575078.805660:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12009
00000400:00000200:1.0:1788575078.805667:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575078.805671:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575078.805674:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575078.805676:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805678:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575078.805680:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805682:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575078.805685:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575078.805686:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575078.805688:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575078.805691:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575078.805692:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575078.805694:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575078.805696:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805697:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575078.805698:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805700:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575078.805702:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575078.805705:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805706:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575078.805709:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575078.805712:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575078.805714:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575078.805715:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575078.805717:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575078.805725:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788575078.805727:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575078.805730:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575078.805731:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575078.805734:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575078.805736:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575078.805740:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575078.805745:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788575078.805751:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575078.805758:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393182464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575078.805766:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575078.805769:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575078.805772:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393182464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575078.805790:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575078.805794:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575078.805797:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575078.805799:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575078.805801:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575078.805804:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575078.805807:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12009
00000400:00000200:1.0:1788575078.805811:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575078.805815:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575078.805817:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575078.805831:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805834:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575078.805836:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805838:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575078.805841:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575078.805842:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575078.805843:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575078.805846:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575078.805848:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575078.805850:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575078.805853:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805856:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575078.805858:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805860:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575078.805863:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575078.805866:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805869:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575078.805871:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575078.805876:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575078.805878:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575078.805881:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575078.805885:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575078.805888:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575078.805890:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575078.805893:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788575078.805895:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575078.805898:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575078.805899:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575078.805901:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575078.805903:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575078.805905:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575078.805909:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788575078.805912:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575078.805917:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393182592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575078.805924:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575078.805926:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575078.805930:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393182592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575078.805939:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575078.805942:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575078.805945:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575078.805947:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575078.805949:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575078.805951:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575078.805955:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12009
00000400:00000200:1.0:1788575078.805958:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575078.805961:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575078.805964:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575078.805966:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805968:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575078.805970:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805972:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575078.805974:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575078.805975:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575078.805977:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575078.805988:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575078.805990:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575078.805993:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575078.805996:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.805998:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575078.806001:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575078.806003:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575078.806005:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575078.806008:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575078.806010:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575078.806012:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575078.806015:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575078.806017:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575078.806019:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575078.806022:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575078.806024:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575078.806026:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575078.806029:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788575078.806032:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575078.806034:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575078.806036:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575078.806038:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575078.806039:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575078.806041:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575078.806041:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:1.0:1788575078.806044:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000800:00000200:2.1:1788575078.806045:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1788575078.806048:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575078.806052:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393182720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575078.806059:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575078.806061:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575078.806065:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393182720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575078.806073:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575078.806076:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575078.806080:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575078.806085:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12020)
00000800:00000200:2.1:1788575078.806413:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575078.806415:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575078.806651:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575078.806654:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1788575079.317481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575079.317487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575079.317489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575079.317493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575079.317496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575079.317499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575079.317501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575079.317503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575079.317506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575079.765503:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306683321, at 4306683328
00000100:00000040:0.0:1788575079.765513:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306683321, at 4306683328
00000100:00000010:1.0:1788575079.765518:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000100:00000010:0.0:1788575079.765523:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000100:00000001:0.0:1788575080.341503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575080.341511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575080.341514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575080.341532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575080.341537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575080.341542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575080.341545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575080.341548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575080.341551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575081.365485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575081.365492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575081.365495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575081.365500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575081.365504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575081.365508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575081.365511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575081.365514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575081.365517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575082.389473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575082.389479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575082.389481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575082.389485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575082.389488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575082.389491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575082.389493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575082.389495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575082.389497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575083.413504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575083.413512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575083.413514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575083.413520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575083.413523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575083.413528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575083.413531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575083.413533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575083.413535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575083.925531:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575083.925542:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575083.925545:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575083.925549:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575083.925554:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575083.925561:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12015
00000400:00000200:0.0:1788575083.925571:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575083.925577:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575083.925581:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575083.925584:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925588:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575083.925590:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925593:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575083.925597:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575083.925599:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575083.925602:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575083.925606:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575083.925609:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575083.925611:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575083.925614:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925617:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575083.925618:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925621:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575083.925624:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575083.925627:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925630:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575083.925634:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575083.925637:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575083.925640:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575083.925642:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575083.925645:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575083.925654:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788575083.925658:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575083.925662:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575083.925663:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575083.925665:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575083.925668:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575083.925671:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575083.925677:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788575083.925682:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575083.925690:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393182848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575083.925706:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575083.925712:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575083.925721:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393182848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575083.925743:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575083.925747:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575083.925750:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575083.925752:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575083.925754:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575083.925756:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575083.925760:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12015
00000400:00000200:0.0:1788575083.925766:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575083.925770:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575083.925773:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575083.925777:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925782:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575083.925788:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925794:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575083.925801:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575083.925804:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575083.925809:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575083.925815:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575083.925821:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575083.925828:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575083.925837:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925842:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575083.925847:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925852:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575083.925858:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575083.925863:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575083.925868:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575083.925873:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575083.925881:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575083.925886:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575083.925892:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575083.925896:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575083.925900:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575083.925902:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575083.925921:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788575083.925925:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575083.925930:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575083.925931:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575083.925935:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575083.925938:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575083.925941:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575083.925947:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788575083.925953:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575083.925962:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393182976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575083.925973:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575083.925978:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575083.925983:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393182976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575083.925996:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575083.926001:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575083.926006:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575083.926009:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575083.926012:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575083.926017:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575083.926024:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12015
00000400:00000200:0.0:1788575083.926035:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575083.926044:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575083.926050:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575083.926059:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.926067:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575083.926073:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.926080:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575083.926086:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575083.926089:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:2.1:1788575083.926089:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:0.0:1788575083.926095:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.1:1788575083.926095:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000040:0.0:1788575083.926100:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575083.926107:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575083.926112:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575083.926121:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.926126:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575083.926133:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575083.926140:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575083.926145:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575083.926154:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575083.926160:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575083.926167:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575083.926173:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575083.926179:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575083.926183:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575083.926188:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575083.926196:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575083.926202:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575083.926208:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788575083.926213:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575083.926220:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575083.926225:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575083.926232:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575083.926235:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575083.926241:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575083.926249:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788575083.926256:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575083.926266:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393183104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575083.926279:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575083.926283:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575083.926290:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393183104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575083.926310:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575083.926314:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575083.926320:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575083.926326:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12025)
00000800:00000200:2.1:1788575083.926561:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575083.926564:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575083.926808:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575083.926810:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1788575084.437552:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575084.437558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575084.437559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575084.437564:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575084.437567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575084.437570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575084.437572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575084.437574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575084.437576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575084.885538:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306688441, at 4306688448
00000100:00000040:1.0:1788575084.885538:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306688441, at 4306688448
00000100:00000010:1.0:1788575084.885555:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000100:00000010:0.0:1788575084.885557:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00000100:00080000:0.0:1788575089.045543:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575089.045555:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575089.045558:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575089.045562:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575089.045567:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575089.045574:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12020
00000400:00000200:0.0:1788575089.045583:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575089.045590:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575089.045595:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575089.045599:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045602:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575089.045606:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045609:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575089.045613:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575089.045615:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575089.045618:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575089.045622:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575089.045625:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575089.045628:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575089.045633:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045635:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575089.045638:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045640:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575089.045645:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575089.045650:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045653:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575089.045657:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575089.045661:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575089.045665:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575089.045667:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575089.045670:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575089.045681:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575089.045685:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575089.045689:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575089.045690:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575089.045694:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575089.045697:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575089.045701:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575089.045710:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788575089.045717:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575089.045726:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393183232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575089.045746:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575089.045751:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575089.045757:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393183232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575089.045777:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575089.045781:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575089.045785:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575089.045788:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575089.045790:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575089.045794:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575089.045797:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12020
00000400:00000200:0.0:1788575089.045802:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575089.045813:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575089.045819:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575089.045831:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045841:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575089.045850:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045861:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575089.045867:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575089.045874:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575089.045882:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575089.045889:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575089.045900:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575089.045908:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575089.045916:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045924:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575089.045932:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045941:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575089.045946:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575089.045953:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575089.045956:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575089.045963:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575089.045971:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575089.045981:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575089.045988:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575089.046009:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575089.046018:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575089.046025:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575089.046032:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788575089.046039:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575089.046044:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575089.046046:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575089.046049:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575089.046055:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575089.046058:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575089.046067:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788575089.046079:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575089.046100:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393183360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575089.046142:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575089.046151:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575089.046172:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393183360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575089.046193:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575089.046204:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575089.046214:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:2.1:1788575089.046215:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575089.046225:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:2.1:1788575089.046225:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00080000:0.0:1788575089.046234:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575089.046245:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575089.046254:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12020
00000400:00000200:0.0:1788575089.046266:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575089.046277:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575089.046284:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575089.046294:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.046301:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575089.046311:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.046316:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575089.046332:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575089.046339:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575089.046346:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575089.046352:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575089.046360:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575089.046367:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575089.046376:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.046381:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575089.046386:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575089.046417:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575089.046421:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575089.046424:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575089.046428:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575089.046430:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575089.046434:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575089.046437:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575089.046440:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575089.046444:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575089.046453:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575089.046459:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575089.046467:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788575089.046475:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575089.046485:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575089.046491:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575089.046494:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575089.046497:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575089.046502:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575089.046509:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788575089.046518:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575089.046529:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393183488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575089.046556:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575089.046564:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575089.046574:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393183488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575089.046599:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575089.046606:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575089.046625:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575089.046636:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12030)
00000800:00000200:2.1:1788575089.046919:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575089.046921:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575089.047198:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575089.047200:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:0.0:1788575090.005497:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306693561, at 4306693568
00000100:00000040:1.0:1788575090.005497:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306693562, at 4306693568
00000100:00000010:1.0:1788575090.005511:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000100:00000010:0.0:1788575090.005515:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000100:00080000:0.0:1788575094.165499:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575094.165506:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575094.165509:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575094.165511:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575094.165514:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575094.165520:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12025
00000400:00000200:0.0:1788575094.165529:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575094.165536:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575094.165539:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575094.165541:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165546:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575094.165549:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165552:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575094.165556:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575094.165558:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575094.165560:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575094.165563:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575094.165566:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575094.165568:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575094.165571:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165573:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575094.165575:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165576:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575094.165578:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575094.165580:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165581:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575094.165583:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575094.165585:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575094.165587:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575094.165588:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575094.165590:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575094.165597:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575094.165600:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575094.165602:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575094.165604:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575094.165606:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575094.165608:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575094.165611:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575094.165616:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788575094.165621:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575094.165628:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393183616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575094.165639:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575094.165642:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575094.165645:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393183616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575094.165658:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575094.165660:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575094.165662:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575094.165663:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575094.165664:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575094.165666:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575094.165668:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12025
00000400:00000200:0.0:1788575094.165671:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575094.165673:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575094.165675:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575094.165680:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165684:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575094.165690:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165694:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575094.165700:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575094.165703:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575094.165708:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575094.165712:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575094.165717:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575094.165724:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575094.165731:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165736:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575094.165742:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165746:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575094.165753:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575094.165769:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165771:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575094.165774:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575094.165779:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575094.165784:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575094.165788:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575094.165793:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575094.165798:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575094.165801:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575094.165810:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788575094.165813:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575094.165815:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575094.165816:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575094.165820:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575094.165822:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575094.165824:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575094.165829:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575094.165834:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575094.165842:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393183744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575094.165857:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575094.165859:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575094.165867:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393183744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575094.165879:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575094.165882:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575094.165885:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575094.165888:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575094.165890:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575094.165893:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575094.165899:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12025
00000800:00000200:2.1:1788575094.165902:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:0.0:1788575094.165906:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:2.1:1788575094.165907:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000400:00000200:0.0:1788575094.165910:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575094.165915:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575094.165921:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165923:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575094.165924:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165926:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575094.165929:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575094.165931:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575094.165934:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575094.165936:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575094.165939:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575094.165943:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575094.165946:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165949:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575094.165952:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165956:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575094.165960:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575094.165964:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575094.165968:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575094.165970:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575094.165974:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575094.165977:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575094.165981:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575094.165983:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575094.165987:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575094.165989:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575094.165992:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788575094.165996:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575094.166001:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575094.166004:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575094.166007:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575094.166010:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575094.166012:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575094.166026:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575094.166033:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575094.166043:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393183872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575094.166055:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575094.166059:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575094.166066:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393183872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575094.166089:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575094.166092:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575094.166094:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575094.166098:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12035)
00000800:00000200:2.1:1788575094.166288:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575094.166290:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575094.166513:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575094.166514:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:0.0:1788575095.125510:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306698681, at 4306698688
00000100:00000010:0.0:1788575095.125524:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000100:00000040:1.0:1788575095.125565:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306698681, at 4306698688
00000100:00000010:1.0:1788575095.125571:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00000800:00000200:1.0:1788575097.237514:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575097.237522:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788575097.237531:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575097.237694:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575097.237700:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575097.237711:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575097.237717:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575097.237721:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:3.1:1788575099.285999:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575099.286003:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788575099.286513:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575099.286516:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788575099.286700:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575099.286701:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:1.0:1788575100.245510:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306703801, at 4306703808
00000100:00000040:0.0:1788575100.245525:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306703801, at 4306703808
00000100:00000010:1.0:1788575100.245530:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788575100.245543:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000001:0.0:1788575102.869550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575102.869562:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575102.869564:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575102.869572:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575102.869576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575102.869580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575102.869584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575102.869587:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575102.869590:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575103.893553:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575103.893562:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575103.893565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575103.893572:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575103.893576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575103.893581:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575103.893584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575103.893588:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575103.893591:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575104.405505:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575104.405519:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575104.405522:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575104.405526:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575104.405531:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575104.405540:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12035
00000400:00000200:1.0:1788575104.405549:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575104.405556:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575104.405559:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575104.405561:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405565:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575104.405567:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405569:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575104.405572:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575104.405574:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575104.405577:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575104.405579:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575104.405581:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575104.405582:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575104.405584:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405585:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575104.405587:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405588:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575104.405591:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575104.405593:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405595:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575104.405597:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575104.405600:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575104.405602:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575104.405604:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575104.405606:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575104.405616:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788575104.405619:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575104.405622:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575104.405623:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575104.405626:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575104.405629:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575104.405633:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575104.405641:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788575104.405647:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575104.405655:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393184384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575104.405668:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575104.405672:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575104.405675:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393184384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575104.405693:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575104.405695:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575104.405698:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575104.405699:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575104.405701:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575104.405703:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575104.405705:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12035
00000400:00000200:1.0:1788575104.405707:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575104.405709:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575104.405711:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575104.405714:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405716:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575104.405717:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405719:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575104.405720:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575104.405721:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575104.405722:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575104.405724:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575104.405727:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575104.405730:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575104.405734:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405738:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575104.405742:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405744:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575104.405748:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575104.405752:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405757:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575104.405760:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575104.405766:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575104.405771:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575104.405775:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575104.405781:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575104.405786:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575104.405789:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575104.405794:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788575104.405797:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575104.405802:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575104.405803:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575104.405804:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575104.405806:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575104.405808:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575104.405813:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788575104.405817:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575104.405823:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393184512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575104.405830:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575104.405831:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575104.405835:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393184512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575104.405841:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575104.405844:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575104.405849:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575104.405851:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575104.405855:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575104.405862:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575104.405868:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12035
00000400:00000200:1.0:1788575104.405877:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575104.405884:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575104.405887:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575104.405891:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405896:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575104.405901:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405906:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575104.405909:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575104.405911:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575104.405913:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575104.405915:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575104.405917:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575104.405920:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575104.405925:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405927:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575104.405932:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405934:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575104.405938:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575104.405943:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575104.405945:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575104.405946:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575104.405949:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575104.405952:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575104.405955:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575104.405959:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575104.405963:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575104.405965:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575104.405970:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788575104.405972:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575104.405974:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575104.405977:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575104.405979:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575104.405980:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575104.405981:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575104.405987:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788575104.405991:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575104.405997:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393184640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1788575104.406003:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788575104.406005:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575104.406009:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575104.406010:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:1.0:1788575104.406013:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393184640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575104.406024:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575104.406027:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575104.406030:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575104.406034:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12045)
00000800:00000200:3.1:1788575104.406335:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575104.406336:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788575104.406560:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575104.406562:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:0.0:1788575104.917511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575104.917519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575104.917521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575104.917526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575104.917529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575104.917533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575104.917536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575104.917538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575104.917540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575105.365518:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306708921, at 4306708928
00000100:00000040:1.0:1788575105.365518:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306708921, at 4306708928
00000100:00000010:1.0:1788575105.365535:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:0.0:1788575105.365538:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000001:0.0:1788575105.941489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575105.941500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575105.941503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575105.941510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575105.941513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575105.941517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575105.941520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575105.941522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575105.941524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575106.965505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575106.965513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575106.965516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575106.965521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575106.965524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575106.965528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575106.965530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575106.965533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575106.965535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575107.989538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575107.989546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575107.989548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575107.989554:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575107.989557:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575107.989561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575107.989564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575107.989567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575107.989569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575109.013498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575109.013502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575109.013504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575109.013507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575109.013510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575109.013512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575109.013514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575109.013516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575109.013518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575109.525497:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575109.525509:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575109.525512:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575109.525517:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575109.525522:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575109.525530:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12040
00000400:00000200:0.0:1788575109.525544:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575109.525551:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575109.525554:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575109.525558:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525562:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575109.525566:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525568:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575109.525573:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575109.525574:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575109.525578:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575109.525580:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575109.525582:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575109.525584:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575109.525586:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525588:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575109.525589:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525590:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575109.525592:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575109.525594:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525596:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575109.525598:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575109.525600:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575109.525602:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575109.525603:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575109.525606:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575109.525612:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788575109.525614:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575109.525617:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575109.525618:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575109.525620:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575109.525622:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575109.525625:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575109.525631:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788575109.525639:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575109.525648:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393184768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575109.525658:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575109.525663:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575109.525667:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393184768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575109.525686:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575109.525688:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575109.525692:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575109.525693:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575109.525695:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575109.525697:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575109.525700:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12040
00000400:00000200:0.0:1788575109.525704:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575109.525707:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575109.525709:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575109.525712:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525715:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575109.525717:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525720:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575109.525723:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575109.525724:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575109.525726:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575109.525730:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575109.525734:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575109.525738:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575109.525744:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525748:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575109.525753:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525757:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575109.525761:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575109.525767:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525771:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575109.525775:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575109.525781:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575109.525786:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575109.525791:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575109.525796:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575109.525803:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575109.525806:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575109.525813:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788575109.525819:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575109.525825:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575109.525829:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575109.525834:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575109.525836:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575109.525838:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575109.525846:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788575109.525853:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575109.525863:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393184896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575109.525877:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575109.525879:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575109.525887:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393184896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575109.525903:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575109.525909:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575109.525924:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575109.525928:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575109.525930:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575109.525934:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575109.525940:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12040
00000400:00000200:0.0:1788575109.525945:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575109.525954:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:3.1:1788575109.525954:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1788575109.525960:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575109.525964:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525966:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575109.525969:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.525974:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575109.525980:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575109.525984:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575109.525988:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575109.525992:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575109.525996:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575109.526001:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575109.526004:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.526008:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575109.526012:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575109.526014:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575109.526018:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575109.526023:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575109.526026:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575109.526029:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575109.526033:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575109.526038:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575109.526048:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575109.526052:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575109.526056:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575109.526058:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575109.526064:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788575109.526067:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575109.526071:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575109.526074:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575109.526075:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575109.526077:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575109.526079:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575109.526082:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788575109.526086:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575109.526093:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393185024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575109.526117:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575109.526119:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575109.526130:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393185024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575109.526140:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575109.526144:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575109.526148:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575109.526154:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12050)
00000800:00000200:3.1:1788575109.526210:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575109.526299:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575110.037533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575110.037540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575110.037541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575110.037546:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575110.037549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575110.037552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575110.037554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575110.037555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575110.037558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575110.485503:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306714041, at 4306714048
00000100:00000010:0.0:1788575110.485515:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000040:1.0:1788575110.485523:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306714041, at 4306714048
00000100:00000010:1.0:1788575110.485537:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000001:0.0:1788575111.061468:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575111.061475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575111.061477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575111.061481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575111.061483:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575111.061486:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575111.061487:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575111.061489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575111.061491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575112.085503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575112.085509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575112.085510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575112.085514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575112.085517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575112.085520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575112.085522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575112.085524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575112.085526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575113.109501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575113.109509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575113.109512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575113.109517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575113.109521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575113.109525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575113.109527:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575113.109530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575113.109533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575114.646260:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575114.646264:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575114.646650:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575114.646652:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575114.646903:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575114.646906:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:0.0:1788575115.605536:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306719161, at 4306719168
00000100:00000040:1.0:1788575115.605536:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306719161, at 4306719168
00000100:00000010:0.0:1788575115.605552:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000100:00000010:1.0:1788575115.605554:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00000020:00000001:1.0:1788575119.125597:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575119.125599:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575119.125602:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788575119.765480:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575119.765489:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575119.765491:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575119.765495:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575119.765499:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575119.765517:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12050
00000400:00000200:0.0:1788575119.765526:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575119.765532:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575119.765535:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575119.765538:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765541:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575119.765543:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765545:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575119.765548:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575119.765550:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575119.765553:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575119.765555:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575119.765558:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575119.765561:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575119.765563:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765565:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575119.765567:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765569:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575119.765572:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575119.765575:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765577:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575119.765579:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575119.765582:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575119.765585:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575119.765587:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575119.765589:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575119.765596:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788575119.765600:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575119.765603:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575119.765605:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575119.765607:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575119.765610:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575119.765613:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575119.765620:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575119.765626:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575119.765635:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393185536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575119.765645:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575119.765649:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575119.765653:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393185536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575119.765672:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575119.765675:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575119.765679:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575119.765681:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575119.765683:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575119.765685:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575119.765688:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12050
00000400:00000200:0.0:1788575119.765692:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575119.765696:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575119.765698:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575119.765703:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765708:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575119.765712:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765716:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575119.765721:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575119.765724:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575119.765727:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575119.765730:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575119.765736:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575119.765741:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575119.765748:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765752:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575119.765757:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765760:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575119.765764:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575119.765769:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765774:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575119.765778:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575119.765785:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575119.765789:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575119.765794:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575119.765800:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575119.765807:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575119.765811:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575119.765818:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575119.765823:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575119.765829:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575119.765831:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575119.765833:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575119.765835:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575119.765838:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575119.765844:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575119.765850:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575119.765860:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393185664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575119.765874:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575119.765878:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575119.765887:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393185664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575119.765907:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575119.765912:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575119.765918:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575119.765922:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575119.765924:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575119.765928:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575119.765933:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12050
00000800:00000200:2.1:1788575119.765934:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575119.765939:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000400:00000200:0.0:1788575119.765941:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575119.765949:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575119.765953:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575119.765959:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765963:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575119.765968:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.765975:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575119.765980:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575119.765983:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575119.765987:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575119.765998:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575119.766003:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575119.766008:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575119.766015:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.766019:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575119.766025:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575119.766029:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575119.766033:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575119.766038:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575119.766041:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575119.766044:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575119.766049:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575119.766054:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575119.766057:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575119.766061:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575119.766065:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575119.766069:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575119.766076:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788575119.766080:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575119.766085:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575119.766087:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575119.766091:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575119.766093:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575119.766097:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575119.766102:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575119.766108:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575119.766116:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393185792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575119.766152:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575119.766156:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575119.766165:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393185792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575119.766181:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575119.766187:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575119.766194:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575119.766201:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12061)
00000800:00000200:2.1:1788575119.766367:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575119.766370:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1788575119.766616:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575119.766618:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000040:0.0:1788575120.725501:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306724281, at 4306724288
00000100:00000040:1.0:1788575120.725502:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306724281, at 4306724288
00000100:00000010:0.0:1788575120.725516:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000100:00000010:1.0:1788575120.725517:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000100:00000001:0.0:1788575121.301522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575121.301527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575121.301529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575121.301555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575121.301558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575121.301561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575121.301563:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575121.301566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575121.301570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575122.197601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575122.197605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575122.197609:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575122.325508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575122.325519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575122.325521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575122.325526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575122.325529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575122.325534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575122.325536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575122.325538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575122.325541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575123.349498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575123.349504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575123.349506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575123.349511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575123.349514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575123.349517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575123.349519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575123.349521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575123.349522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575124.373506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575124.373511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575124.373512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575124.373516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575124.373518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575124.373521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575124.373522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575124.373524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575124.373526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575124.886042:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575124.886048:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788575124.886495:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575124.886497:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788575124.886682:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575124.886683:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:1.0:1788575125.269553:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575125.269556:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575125.269559:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575125.397472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575125.397478:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575125.397480:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575125.397483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575125.397486:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575125.397488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575125.397490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575125.397492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575125.397494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575125.845524:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306729401, at 4306729408
00000100:00000040:1.0:1788575125.845524:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306729401, at 4306729408
00000100:00000010:1.0:1788575125.845542:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000100:00000010:0.0:1788575125.845545:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000100:00000001:0.0:1788575126.421479:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575126.421488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575126.421489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575126.421495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575126.421498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575126.421502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575126.421504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575126.421507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575126.421509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575127.445603:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575127.445615:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575127.445618:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575127.445627:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575127.445631:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575127.445660:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575127.445663:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575127.445667:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575127.445670:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575128.341601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575128.341606:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575128.341610:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575128.469527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575128.469536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575128.469538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575128.469543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575128.469546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575128.469549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575128.469552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575128.469555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575128.469557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575129.493556:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575129.493563:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575129.493565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575129.493570:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575129.493573:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575129.493577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575129.493579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575129.493581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575129.493583:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575130.006140:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575130.006145:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788575130.006585:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575130.006586:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788575130.006867:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575130.006869:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:0.0:1788575130.517574:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575130.517582:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575130.517584:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575130.517590:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575130.517593:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575130.517597:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575130.517599:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575130.517601:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575130.517604:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575130.965482:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306734521, at 4306734528
00000100:00000040:1.0:1788575130.965482:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306734521, at 4306734528
00000100:00000010:1.0:1788575130.965500:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000100:00000010:0.0:1788575130.965502:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000020:00000001:1.0:1788575131.413605:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575131.413609:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575131.413612:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575131.541631:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575131.541641:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575131.541645:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575131.541653:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575131.541658:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575131.541663:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575131.541666:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575131.541670:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575131.541675:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575132.565527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575132.565535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575132.565537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575132.565543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575132.565546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575132.565550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575132.565553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575132.565556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575132.565560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575133.589518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575133.589529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575133.589531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575133.589538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575133.589543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575133.589547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575133.589550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575133.589553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575133.589556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575134.485542:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575134.485545:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575134.485549:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575134.613549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575134.613558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575134.613561:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575134.613569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575134.613573:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575134.613577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575134.613580:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575134.613582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575134.613586:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788575135.125530:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788575135.125543:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788575135.125550:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788575135.125788:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1788575135.125795:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788575135.125817:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788575135.125825:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575135.125831:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:2.1:1788575135.126024:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575135.126028:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788575135.126437:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575135.126439:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788575135.126739:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575135.126742:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:0.0:1788575136.085500:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306739641, at 4306739648
00000100:00000010:0.0:1788575136.085516:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000100:00000040:1.0:1788575136.085585:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306739641, at 4306739648
00000100:00000010:1.0:1788575136.085590:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000020:00000001:1.0:1788575137.557738:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575137.557743:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575137.557746:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1788575140.246102:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575140.246108:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575140.246628:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575140.246630:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575140.246940:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575140.246942:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:1.0:1788575140.629670:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575140.629674:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575140.629677:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1788575141.205501:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306744761, at 4306744768
00000100:00000040:1.0:1788575141.205501:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306744761, at 4306744768
00000100:00000010:1.0:1788575141.205532:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000100:00000010:0.0:1788575141.205535:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000020:00000001:1.0:1788575143.701573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575143.701577:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575143.701580:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1788575145.365493:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575145.365500:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575145.365502:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575145.365506:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575145.365509:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575145.365514:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12076
00000400:00000200:1.0:1788575145.365521:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575145.365527:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575145.365530:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575145.365533:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365535:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575145.365538:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365541:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575145.365544:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575145.365547:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575145.365549:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575145.365552:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575145.365555:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575145.365557:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575145.365559:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365561:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575145.365567:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365569:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575145.365571:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575145.365573:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365575:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575145.365577:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575145.365579:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575145.365581:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575145.365582:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575145.365584:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575145.365591:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788575145.365594:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575145.365596:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575145.365597:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575145.365600:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575145.365601:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575145.365605:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575145.365610:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788575145.365615:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575145.365621:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393187456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575145.365628:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575145.365631:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575145.365634:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393187456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575145.365647:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575145.365650:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575145.365652:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575145.365653:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575145.365657:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575145.365659:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575145.365661:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12076
00000400:00000200:1.0:1788575145.365664:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575145.365666:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575145.365667:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575145.365669:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365671:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575145.365672:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365673:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575145.365675:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575145.365676:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575145.365677:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575145.365678:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575145.365680:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575145.365681:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575145.365683:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365684:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575145.365686:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365688:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575145.365689:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575145.365691:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365693:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575145.365696:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575145.365700:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575145.365704:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575145.365707:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575145.365710:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575145.365713:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575145.365716:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575145.365720:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788575145.365723:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575145.365727:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575145.365729:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575145.365733:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575145.365734:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575145.365737:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575145.365743:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788575145.365746:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575145.365756:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393187584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575145.365773:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575145.365777:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575145.365787:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393187584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575145.365799:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575145.365803:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575145.365806:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575145.365809:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575145.365812:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575145.365813:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575145.365815:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12076
00000400:00000200:1.0:1788575145.365820:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575145.365823:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575145.365824:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575145.365827:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365830:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575145.365833:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365837:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575145.365842:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575145.365843:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575145.365844:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575145.365847:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575145.365849:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575145.365853:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575145.365867:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365872:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575145.365877:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365881:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575145.365885:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575145.365891:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575145.365894:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575145.365899:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575145.365903:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575145.365904:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575145.365908:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575145.365911:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575145.365913:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575145.365914:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575145.365918:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788575145.365922:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575145.365926:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575145.365929:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575145.365938:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575145.365940:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575145.365943:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575145.365948:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788575145.365953:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575145.365959:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393187712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575145.365972:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575145.365975:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575145.365981:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393187712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575145.365987:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575145.365992:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575145.365995:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575145.366000:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12086)
00000800:00000200:2.1:1788575145.366011:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575145.366014:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575145.366432:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575145.366434:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575145.366725:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575145.366727:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:0.0:1788575146.325466:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306749881, at 4306749888
00000100:00000040:1.0:1788575146.325466:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306749881, at 4306749888
00000100:00000010:1.0:1788575146.325483:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000100:00000010:0.0:1788575146.325485:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00000020:00000001:1.0:1788575146.773562:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575146.773565:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575146.773567:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1788575149.845538:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575149.845541:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575149.845543:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575149.973497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575149.973503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575149.973505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575149.973509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575149.973517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575149.973520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575149.973523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575149.973525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575149.973527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575150.486098:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575150.486104:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575150.486520:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575150.486521:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575150.486747:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575150.486748:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1788575150.997540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575150.997548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575150.997550:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575150.997556:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575150.997561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575150.997565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575150.997568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575150.997571:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575150.997575:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575151.445493:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306755001, at 4306755008
00000100:00000040:1.0:1788575151.445493:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306755001, at 4306755008
00000100:00000010:1.0:1788575151.445507:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000100:00000010:0.0:1788575151.445511:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00000100:00000001:1.0:1788575152.021504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575152.021512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575152.021514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575152.021520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575152.021531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575152.021534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575152.021536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575152.021538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575152.021540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575152.917607:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575152.917612:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575152.917615:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575153.045476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575153.045484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575153.045487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575153.045493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575153.045496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575153.045500:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575153.045503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575153.045506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575153.045509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575154.069515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575154.069522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575154.069524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575154.069530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575154.069534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575154.069538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575154.069541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575154.069544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575154.069547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575155.093468:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575155.093474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575155.093476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575155.093480:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575155.093483:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575155.093485:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575155.093488:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575155.093489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575155.093492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575155.606195:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575155.606203:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575155.606649:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575155.606652:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575155.606876:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575155.606878:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:1.0:1788575155.989556:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575155.989559:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575155.989562:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575156.117486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575156.117494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575156.117497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575156.117503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575156.117506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575156.117510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575156.117513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575156.117516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575156.117519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575156.565553:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306760121, at 4306760128
00000100:00000040:0.0:1788575156.565554:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306760121, at 4306760128
00000100:00000010:1.0:1788575156.565595:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000100:00000010:0.0:1788575156.565611:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00000100:00000001:1.0:1788575157.141488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575157.141497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575157.141500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575157.141508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575157.141511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575157.141525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575157.141533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575157.141536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575157.141541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575158.165532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575158.165537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575158.165539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575158.165543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575158.165546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575158.165550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575158.165552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575158.165555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575158.165557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575159.061700:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575159.061704:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575159.061707:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575159.189518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575159.189527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575159.189529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575159.189535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575159.189538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575159.189542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575159.189545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575159.189548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575159.189551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575160.213540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575160.213551:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575160.213554:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575160.213562:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575160.213567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575160.213573:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575160.213576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575160.213580:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575160.213584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575160.725511:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575160.725522:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575160.725525:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575160.725529:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575160.725534:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575160.725541:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12091
00000400:00000200:0.0:1788575160.725550:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575160.725558:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575160.725560:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575160.725563:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725565:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575160.725567:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725569:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575160.725573:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575160.725575:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575160.725578:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575160.725580:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575160.725582:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575160.725584:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575160.725585:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725587:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575160.725588:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725590:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575160.725593:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575160.725596:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725597:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575160.725600:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575160.725602:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575160.725605:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575160.725606:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575160.725609:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575160.725619:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788575160.725622:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575160.725625:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575160.725626:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575160.725629:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575160.725632:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575160.725636:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575160.725644:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788575160.725652:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575160.725661:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393188608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575160.725670:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575160.725674:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575160.725677:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393188608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575160.725697:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575160.725699:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575160.725702:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575160.725703:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575160.725705:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575160.725707:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575160.725709:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12091
00000400:00000200:0.0:1788575160.725711:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575160.725713:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575160.725715:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575160.725718:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725719:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575160.725721:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725723:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575160.725725:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575160.725727:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575160.725729:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575160.725731:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575160.725734:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575160.725737:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575160.725740:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725744:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575160.725750:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725756:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575160.725762:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575160.725769:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725776:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575160.725784:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575160.725792:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575160.725796:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575160.725797:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575160.725802:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575160.725806:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575160.725812:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575160.725823:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575160.725827:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575160.725835:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575160.725837:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575160.725842:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575160.725845:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575160.725848:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575160.725864:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788575160.725869:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575160.725877:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393188736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575160.725884:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575160.725890:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575160.725898:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393188736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575160.725916:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575160.725918:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575160.725920:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575160.725922:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575160.725925:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575160.725927:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575160.725931:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12091
00000400:00000200:0.0:1788575160.725943:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575160.725948:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575160.725950:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575160.725962:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725964:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575160.725967:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725971:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575160.725975:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575160.725979:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575160.725981:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575160.725984:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575160.725988:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575160.725991:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575160.725994:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.725996:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575160.726003:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575160.726009:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575160.726015:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575160.726021:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575160.726025:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575160.726030:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575160.726035:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575160.726039:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575160.726045:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575160.726049:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575160.726053:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575160.726060:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575160.726064:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788575160.726069:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575160.726071:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575160.726073:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575160.726074:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575160.726077:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575160.726079:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575160.726085:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788575160.726094:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575160.726099:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393188864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575160.726115:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575160.726118:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575160.726127:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393188864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575160.726134:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575160.726139:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575160.726152:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575160.726157:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12101)
00000800:00000200:2.1:1788575160.726232:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575160.726235:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575160.726755:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575160.726758:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575160.727077:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575160.727080:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788575161.237558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575161.237568:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575161.237571:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575161.237579:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575161.237583:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575161.237588:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575161.237591:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575161.237595:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575161.237598:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575161.685537:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306765241, at 4306765248
00000100:00000040:1.0:1788575161.685537:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306765241, at 4306765248
00000100:00000010:0.0:1788575161.685556:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000100:00000010:1.0:1788575161.685560:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000020:00000001:1.0:1788575162.133614:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575162.133619:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575162.133623:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575162.261509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575162.261515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575162.261516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575162.261521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575162.261523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575162.261526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575162.261528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575162.261531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575162.261533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575163.285534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575163.285546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575163.285549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575163.285558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575163.285564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575163.285569:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575163.285572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575163.285576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575163.285580:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575164.309489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575164.309495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575164.309498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575164.309503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575164.309506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575164.309509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575164.309512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575164.309513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575164.309516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575165.205562:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575165.205566:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575165.205569:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575165.333536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575165.333549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575165.333551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575165.333557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575165.333560:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575165.333564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575165.333567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575165.333570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575165.333573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575165.846086:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575165.846091:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1788575165.846601:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575165.846603:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1788575165.846849:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575165.846851:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00000001:1.0:1788575166.357505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575166.357513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575166.357515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575166.357522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575166.357526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575166.357530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575166.357533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575166.357535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575166.357539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575166.805483:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306770361, at 4306770368
00000100:00000040:1.0:1788575166.805483:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306770361, at 4306770368
00000100:00000010:0.0:1788575166.805497:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:1.0:1788575166.805500:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000001:1.0:1788575167.381473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575167.381480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575167.381482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575167.381486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575167.381488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575167.381491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575167.381493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575167.381495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575167.381497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575168.277576:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575168.277580:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575168.277584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575168.405476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575168.405484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575168.405486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575168.405492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575168.405496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575168.405506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575168.405508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575168.405511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575168.405514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575169.429472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575169.429480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575169.429483:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575169.429488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575169.429492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575169.429496:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575169.429499:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575169.429501:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575169.429504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575170.453518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575170.453526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575170.453528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575170.453534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575170.453539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575170.453543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575170.453546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575170.453548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575170.453551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575170.965519:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575170.965528:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575170.965531:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575170.965534:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575170.965538:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575170.965544:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12102
00000400:00000200:0.0:1788575170.965550:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575170.965556:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575170.965558:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575170.965561:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965563:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575170.965565:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965567:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575170.965570:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575170.965571:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575170.965573:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575170.965575:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575170.965577:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575170.965578:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575170.965580:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965581:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575170.965582:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965584:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575170.965586:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575170.965589:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965591:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575170.965593:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575170.965595:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575170.965597:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575170.965598:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575170.965600:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575170.965605:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788575170.965608:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575170.965610:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575170.965611:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575170.965613:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575170.965615:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575170.965618:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575170.965624:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575170.965630:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575170.965637:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393189376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575170.965644:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575170.965648:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575170.965651:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393189376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575170.965666:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575170.965669:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575170.965671:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575170.965672:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575170.965674:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575170.965675:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575170.965678:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12102
00000400:00000200:0.0:1788575170.965681:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575170.965683:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575170.965685:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575170.965688:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965690:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575170.965695:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965696:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575170.965700:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575170.965703:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575170.965720:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575170.965724:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575170.965729:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575170.965734:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575170.965740:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965746:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575170.965749:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965752:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575170.965754:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575170.965756:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965758:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575170.965763:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575170.965770:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575170.965775:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575170.965779:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575170.965783:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575170.965786:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575170.965789:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575170.965794:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788575170.965799:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575170.965802:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575170.965804:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575170.965807:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575170.965809:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575170.965811:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575170.965819:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788575170.965822:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575170.965829:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393189504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575170.965842:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575170.965847:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575170.965857:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393189504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575170.965879:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575170.965886:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575170.965896:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575170.965901:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575170.965906:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575170.965912:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575170.965922:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12102
00000400:00000200:0.0:1788575170.965934:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575170.965944:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575170.965951:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575170.965960:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965967:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575170.965974:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.965979:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575170.965985:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575170.965989:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575170.965992:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.1:1788575170.965995:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1788575170.965997:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:3.1:1788575170.966000:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:0.0:1788575170.966003:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575170.966010:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575170.966024:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.966027:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575170.966032:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575170.966035:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575170.966037:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575170.966043:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575170.966049:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575170.966053:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575170.966058:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575170.966063:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575170.966066:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575170.966069:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575170.966074:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575170.966078:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575170.966084:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788575170.966087:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575170.966089:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575170.966091:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575170.966095:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575170.966098:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575170.966103:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575170.966112:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788575170.966117:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575170.966123:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393189632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575170.966134:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575170.966138:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575170.966144:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393189632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575170.966157:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575170.966163:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575170.966170:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575170.966178:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12112)
00000800:00000200:3.1:1788575170.966495:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575170.966497:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1788575170.966735:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575170.966738:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:1.0:1788575171.349559:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575171.349564:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575171.349567:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575171.477473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575171.477479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575171.477480:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575171.477485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575171.477488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575171.477491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575171.477493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575171.477496:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575171.477498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575171.925490:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306775481, at 4306775488
00000100:00000040:0.0:1788575171.925494:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306775481, at 4306775488
00000100:00000010:1.0:1788575171.925503:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:0.0:1788575171.925507:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000001:1.0:1788575172.501508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575172.501515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575172.501518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575172.501523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575172.501527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575172.501531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575172.501533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575172.501536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575172.501538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575173.525487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575173.525493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575173.525495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575173.525499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575173.525502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575173.525505:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575173.525508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575173.525509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575173.525512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788575174.037508:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788575174.037514:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788575174.037546:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788575174.037680:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1788575174.037692:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788575174.037694:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788575174.037706:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575174.037709:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1788575174.421543:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575174.421546:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575174.421549:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575174.549486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575174.549505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575174.549507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575174.549511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575174.549514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575174.549517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575174.549519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575174.549521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575174.549523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575175.573573:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575175.573584:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575175.573587:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575175.573595:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575175.573600:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575175.573605:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575175.573609:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575175.573612:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575175.573616:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575176.086102:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575176.086341:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000400:00000010:3.0:1788575176.086476:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526330.
00000100:00000200:3.0:1788575176.086479:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393190016, offset 0
00000400:00000200:3.0:1788575176.086483:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575176.086488:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575176.086491:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575176.086495:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575176.086497:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575176.086499:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575176.086501:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575176.086504:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330740:330740:254:4294967295] 192.168.202.110@tcp LPNI seq info [330697:330697:6:4294967295]
00000400:00000200:3.0:1788575176.086508:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575176.086514:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575176.086518:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575176.086522:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b800.
00000800:00000200:3.0:1788575176.086525:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575176.086529:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575176.086532:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575176.086535:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575176.086538:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575176.086540:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306779833
00000100:00000001:3.0:1788575176.086542:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306780601
00000100:00000001:3.0:1788575176.086544:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306779833
00000100:00000001:3.0:1788575176.086546:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306780601
00000100:00000001:3.0:1788575176.086549:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575176.086551:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575176.086553:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575176.086555:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575176.086559:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575176.086560:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1788575176.086562:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575176.086563:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575176.086565:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575176.086567:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575176.469540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575176.469550:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575176.469553:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575176.469555:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575176.469560:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575176.469562:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575176.469564:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575176.469565:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575176.469568:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575176.469569:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575176.469572:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575176.469573:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575177.045461:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306780601, at 4306780608
00000100:00000040:1.0:1788575177.045461:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306780601, at 4306780608
00000100:00000010:0.0:1788575177.045482:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:1.0:1788575177.045485:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575177.493505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575177.493512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575177.493515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575177.493516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575177.493519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575177.493520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575177.493582:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575177.493585:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575177.493589:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00000001:3.0:1788575178.517513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575178.517520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575178.517525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575178.517527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575178.517530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575178.517531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788575179.157588:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788575179.157620:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788575179.157626:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575179.157629:0:2351:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788575179.157632:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575179.157635:0:2351:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788575179.157638:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575179.157640:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:1.0:1788575179.157644:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575179.157648:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575179.157655:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526000
00000400:00000010:1.0:1788575179.157658:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526000.
00000100:00000001:1.0:1788575179.157663:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575179.157672:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e00a80 x1875444393189760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575179.157695:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575179.157700:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393189760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575179.157707:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575179.157709:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575179.157719:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575179.157722:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b600.
00000400:00000200:1.0:1788575179.157725:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575179.157728:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575179.157731:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526e58
00000400:00000010:1.0:1788575179.157732:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526e58.
00000100:00000001:1.0:1788575179.157734:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575179.157738:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e00e00 x1875444393189888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.157740:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575179.157745:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575179.157748:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1788575179.157749:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:1.0:1788575179.157754:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393189888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575179.157760:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575176/real 1788575179]  req@ffff895348e00e00 x1875444393189888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575201 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575179.157764:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575179.157768:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575179.157777:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000400:00000001:3.0:1788575179.157779:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000800:00000010:1.0:1788575179.157780:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b800.
00000400:00000001:3.0:1788575179.157784:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000200:1.0:1788575179.157786:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000010:3.0:1788575179.157789:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:1.0:1788575179.157790:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575179.157797:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526330
00000400:00000200:3.0:1788575179.157797:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:1.0:1788575179.157802:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526330.
00000400:00000010:3.0:1788575179.157802:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:1.0:1788575179.157808:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575179.157812:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e01c00 x1875444393190016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.157818:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000001:1.0:1788575179.157819:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575179.157829:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393190016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575179.157829:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393189888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575179.157856:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575179.157860:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000200:3.0:1788575179.157870:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393189888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.157878:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575179.157880:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.157883:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575179.157885:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575179.157890:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393189888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.157898:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575179.157903:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393189888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.157910:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575179.157913:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.157915:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575179.157916:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.157919:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575179.157921:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575179.157925:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575179.157930:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575179.157933:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575179.157937:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393189888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575179.157945:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393189888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575179.157949:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575179.157953:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575176/real 1788575179]  req@ffff895348e01c00 x1875444393190016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575179.157966:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575179.157967:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575179.157969:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c52c0.
00000400:00000200:3.0:1788575179.157972:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575179.157974:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575179.157976:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575179.157979:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393190016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575179.157988:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393190016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.157994:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575179.157996:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.157998:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575179.157999:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575179.158002:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393190016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.158008:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575179.158011:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393190016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.158017:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575179.158019:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158020:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575179.158022:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158023:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575179.158025:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575179.158027:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575179.158031:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575179.158033:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575179.158037:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393190016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575179.158044:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393190016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575179.158055:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575179.158058:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575176/real 1788575179]  req@ffff895348e00a80 x1875444393189760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575179.158065:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575179.158067:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575179.158069:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575179.158071:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575179.158073:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575179.158076:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575179.158079:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393189760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575179.158085:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393189760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.158090:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575179.158091:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158093:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575179.158094:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575179.158097:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393189760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.158102:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575179.158106:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393189760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.158111:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575179.158113:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158115:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575179.158117:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158118:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575179.158120:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575179.158122:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575179.158125:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575179.158127:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575179.158131:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393189760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575179.158138:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393189760:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575179.158141:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158143:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575179.158146:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393189888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.158152:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575179.158155:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575179.158160:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575179.158164:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575179.158167:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575179.158169:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575179.158171:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575179.158174:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575179.158178:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575179.158179:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575179.158181:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:3.0:1788575179.158185:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575179.158186:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158188:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575179.158191:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393190016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.158198:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575179.158199:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575179.158201:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575179.158204:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575179.158205:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575179.158207:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575179.158219:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575179.158221:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575179.158223:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575179.158224:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575179.158226:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:3.0:1788575179.158228:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575179.158229:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158230:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575179.158233:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393189760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575179.158239:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575179.158240:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575179.158241:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575179.158244:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575179.158245:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575179.158246:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575179.158248:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575179.158249:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575179.158251:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575179.158252:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575179.158254:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575179.158256:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575179.158257:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158260:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575179.158270:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575179.158272:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575179.158274:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575179.158276:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575179.541462:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575179.541467:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575179.541479:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575179.541480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575179.541482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575179.541483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575180.565495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575180.565502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575180.565508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575180.565509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575180.565512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575180.565514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575180.565571:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575180.565575:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575180.565579:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788575181.205484:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575181.205492:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575181.205495:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575181.205498:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575181.205502:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575181.205509:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12112
00000400:00000200:0.0:1788575181.205517:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575181.205523:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575181.205525:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575181.205528:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205531:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575181.205534:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205536:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575181.205540:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575181.205542:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575181.205544:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575181.205547:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575181.205550:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575181.205552:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575181.205555:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205557:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575181.205559:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205560:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575181.205563:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575181.205566:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205569:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575181.205571:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575181.205574:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575181.205577:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575181.205578:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575181.205581:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575181.205597:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788575181.205600:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575181.205605:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575181.205606:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575181.205609:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575181.205612:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575181.205615:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575181.205622:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575181.205629:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575181.205638:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575181.205648:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575181.205652:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575181.205657:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575181.205678:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575181.205682:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575181.205685:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575181.205686:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575181.205688:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575181.205691:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575181.205693:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12112
00000400:00000200:0.0:1788575181.205697:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575181.205700:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575181.205702:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575181.205706:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205708:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575181.205711:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205713:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575181.205715:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575181.205717:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575181.205719:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575181.205720:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1788575181.205722:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575181.205725:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788575181.205726:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575181.205730:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575181.205733:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205734:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575181.205736:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575181.205736:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788575181.205740:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575181.205742:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:3.0:1788575181.205746:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788575181.205747:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205753:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575181.205757:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575181.205762:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788575181.205764:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575181.205765:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575181.205770:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575181.205771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575181.205773:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575181.205775:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:0.0:1788575181.205778:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575181.205780:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788575181.205783:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575181.205787:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393190144:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575181.205788:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575181.205794:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000100:00000001:3.0:1788575181.205795:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1788575181.205799:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575181.205801:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575181.205803:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575181.205805:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575181.205806:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575181.205810:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575181.205810:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1788575181.205813:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575181.205816:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575181.205817:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575181.205821:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575181.205822:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575181.205827:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575181.205833:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575181.205842:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575181.205842:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575181.205847:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575181.205850:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000040:0.0:1788575181.205853:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788575181.205859:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000001:0.0:1788575181.205867:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575181.205868:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393190144, portal 25
00000100:00080000:0.0:1788575181.205870:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575181.205873:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1788575181.205876:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575181.205876:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523503232 : -130484186048384 : ffff895348711880)
00000100:00000001:0.0:1788575181.205880:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575181.205883:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575181.205888:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:3.0:1788575181.205888:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788575181.205893:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12112
00000400:00000200:0.0:1788575181.205901:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575181.205904:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:0.0:1788575181.205908:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1788575181.205910:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788575181.205913:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1788575181.205915:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000040:0.0:1788575181.205919:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:3.0:1788575181.205921:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393190144, offset 0
00000100:00000001:0.0:1788575181.205924:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575181.205927:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575181.205930:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575181.205932:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788575181.205934:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575181.205936:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575181.205938:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575181.205940:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575181.205943:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575181.205944:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1788575181.205948:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575181.205953:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575181.205955:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788575181.205956:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575181.205961:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575181.205963:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575181.205964:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575181.205968:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575181.205969:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1788575181.205973:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575181.205977:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788575181.205978:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575181.205982:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575181.205983:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:0.0:1788575181.205986:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575181.205990:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330741:330741:256:4294967295] 192.168.202.110@tcp LPNI seq info [330698:330698:8:4294967295]
02000000:00000001:0.0:1788575181.205991:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575181.205996:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575181.205999:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:0.0:1788575181.206000:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575181.206004:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575181.206006:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575181.206011:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000400:00000200:3.0:1788575181.206012:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:0.0:1788575181.206016:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575181.206020:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575181.206022:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575181.206024:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575181.206025:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575181.206027:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575181.206030:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575181.206033:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
02000000:00000010:0.0:1788575181.206035:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575181.206040:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575181.206043:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:0.0:1788575181.206048:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.0:1788575181.206050:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1788575181.206062:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1788575181.206062:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575181.206066:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575181.206068:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:0.0:1788575181.206073:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:3.0:1788575181.206078:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575181.206084:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575181.206100:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1788575181.206103:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575181.206107:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575181.206108:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1788575181.206115:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1788575181.206117:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000100:00000040:0.0:1788575181.206122:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12122)
00000400:00000010:3.0:1788575181.206122:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575181.206192:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575181.206197:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206206:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575181.206209:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206216:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575181.206217:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575181.206224:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575181.206228:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575181.206231:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306785721
00000100:00000010:3.0:1788575181.206236:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575181.206243:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575181.206257:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306785721
00000100:00000010:3.0:1788575181.206260:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575181.206263:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575181.206266:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575181.206268:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575181.206271:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575181.206275:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575181/real 1788575181]  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575181.206285:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575181.206286:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575181.206289:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575181.206291:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575181.206293:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575181.206295:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575181.206298:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575181.206305:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206311:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575181.206312:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206315:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575181.206317:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575181.206321:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206327:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575181.206332:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206343:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575181.206346:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206347:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575181.206349:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206351:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575181.206353:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575181.206355:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575181.206359:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575181.206361:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575181.206365:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575181.206373:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393190144:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575181.206378:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206380:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575181.206382:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711880 x1875444393190144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206389:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575181.206416:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575181.206419:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575181.206421:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575181.206423:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575181.206424:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575181.206426:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575181.206429:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:3.0:1788575181.206432:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575181.206433:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575181.206435:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711880.
00000100:00000001:3.0:1788575181.206438:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575181.206440:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206442:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206450:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575181.206452:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575181.206454:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575181.206456:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575181.206457:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575181.206461:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206468:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575181.206469:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575181.206471:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575181.206473:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575181.206475:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575181.206477:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393190272:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575181.206480:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575181.206482:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575181.206489:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575181.206491:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575181.206494:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575181.206495:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575181.206500:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575181.206503:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575181.206506:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575181.206510:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393190272, portal 10
00000100:00000001:3.0:1788575181.206512:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575181.206513:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523511296 : -130484186040320 : ffff895348713800)
00000100:00000040:3.0:1788575181.206519:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206524:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575181.206526:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575181.206528:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575181.206531:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393190272, offset 0
00000400:00000200:3.0:1788575181.206534:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575181.206538:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575181.206542:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575181.206545:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575181.206548:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575181.206550:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575181.206553:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575181.206555:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330742:330742:256:4294967295] 192.168.202.110@tcp LPNI seq info [330699:330699:8:4294967295]
00000400:00000200:3.0:1788575181.206560:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575181.206566:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575181.206569:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575181.206572:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575181.206574:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575181.206576:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575181.206579:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575181.206581:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575181.206584:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575181.206586:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575181.206589:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575181.206592:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575181.206595:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575181.206597:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575181.206599:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575181.206602:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206610:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575181.206613:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206620:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575181.206622:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575181.206625:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575181.206629:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575181.206633:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575181.206635:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306784807
00000100:00000001:3.0:1788575181.206638:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306785721
00000100:00000001:3.0:1788575181.206641:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306784807
00000100:00000001:3.0:1788575181.206642:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306785721
00000100:00000001:3.0:1788575181.206644:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575181.206647:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575181.206649:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575181.206653:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575181/real 1788575181]  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575181.206660:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575181.206662:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575181.206663:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575181.206665:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575181.206667:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575181.206669:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575181.206672:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575181.206680:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206685:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575181.206686:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206688:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575181.206689:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575181.206692:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206698:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575181.206702:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206707:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575181.206709:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206711:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575181.206712:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206713:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575181.206716:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575181.206718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575181.206721:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575181.206722:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575181.206725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575181.206732:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393190272:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575181.206735:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575181.206737:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206742:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575181.206744:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575181.206746:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575181.206747:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575181.206749:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575181.206751:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393190400:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575181.206754:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575181.206755:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575181.206756:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575181.206758:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575181.206761:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575181.206763:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575181.206766:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575181.206769:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575181.206771:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575181.206774:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393190400, portal 4
00000100:00000001:3.0:1788575181.206775:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575181.206776:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:3.0:1788575181.206781:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206786:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575181.206788:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575181.206789:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575181.206791:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393190400, offset 0
00000400:00000200:3.0:1788575181.206794:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575181.206797:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575181.206800:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575181.206803:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575181.206805:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575181.206807:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575181.206809:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575181.206812:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330743:330743:256:4294967295] 192.168.202.110@tcp LPNI seq info [330700:330700:8:4294967295]
00000400:00000200:3.0:1788575181.206815:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575181.206820:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575181.206823:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575181.206826:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575181.206830:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575181.206832:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575181.206835:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575181.206838:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575181.206841:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575181.206842:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575181.206845:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575181.206847:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575181.206850:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575181.206852:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575181.206854:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575181.206856:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206864:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575181.206867:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206873:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575181.206874:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575181.206878:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575181.206881:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575181.206884:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575181.206886:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306784953
00000100:00000001:3.0:1788575181.206888:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306785721
00000100:00000001:3.0:1788575181.206890:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306784953
00000100:00000001:3.0:1788575181.206892:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306785721
00000100:00000001:3.0:1788575181.206894:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575181.206896:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575181.206898:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575181.206901:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575181/real 1788575181]  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575181.206908:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575181.206909:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575181.206910:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575181.206912:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575181.206914:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575181.206915:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575181.206918:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575181.206924:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206930:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575181.206931:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206932:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575181.206933:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575181.206937:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206942:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575181.206945:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206951:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575181.206953:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206954:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575181.206955:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206957:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575181.206959:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575181.206961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575181.206964:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575181.206966:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575181.206970:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575181.206977:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393190400:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575181.206981:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.206983:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575181.206986:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713800 x1875444393190272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.206990:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575181.206991:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575181.206992:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575181.206994:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575181.206995:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575181.206996:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575181.206997:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575181.206999:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:3.0:1788575181.207000:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575181.207001:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575181.207002:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713800.
00000100:00000001:3.0:1788575181.207004:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575181.207005:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.207006:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575181.207008:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393190400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575181.207012:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575181.207012:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575181.207013:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575181.207014:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575181.207015:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575181.207015:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575181.207016:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575181.207017:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:3.0:1788575181.207018:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575181.207019:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575181.207020:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:3.0:1788575181.207021:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575181.207022:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.207024:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575181.207028:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575181.207029:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575181.207030:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575181.207032:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575181.589497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575181.589503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575181.589509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575181.589510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575181.589514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575181.589515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575182.165468:0:216380:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306785721, at 4306785728
00000100:00000040:1.0:1788575182.165468:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306785721, at 4306785728
00000100:00000010:1.0:1788575182.165481:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575182.165484:0:216380:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00010000:00000001:3.0:1788575182.613483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575182.613488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575182.613492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575182.613492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575182.613495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575182.613496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575183.637492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575183.637498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575183.637503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575183.637506:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575183.637509:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575183.637512:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575183.637515:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575183.637517:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575183.637519:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575183.637521:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575183.637524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575183.637526:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575183.637527:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575183.637528:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575183.637530:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575183.637531:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575183.637533:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575183.637535:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575183.637597:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575183.637601:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575183.637604:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00000001:3.0:1788575184.661476:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575184.661481:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575184.661492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575184.661493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575184.661496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575184.661498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575185.685497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575185.685502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575185.685507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575185.685509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575185.685513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575185.685515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575186.325687:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575186.325692:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575186.325696:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575186.325704:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.325716:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575186.325721:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575186.325726:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575186.325738:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575186.325744:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575186.325752:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393190528:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575186.325760:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575186.325765:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575186.325768:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575186.325772:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575186.325797:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575186.325800:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575186.325808:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575186.325813:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575186.325822:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575186.325829:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393190528, portal 25
00000100:00000001:3.0:1788575186.325834:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575186.325836:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:3.0:1788575186.325843:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.325859:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575186.325868:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575186.325873:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575186.325880:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393190528, offset 0
00000400:00000200:3.0:1788575186.325892:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575186.325905:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575186.325916:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575186.325925:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575186.325931:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575186.325937:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575186.325942:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575186.325947:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330744:330744:256:4294967295] 192.168.202.110@tcp LPNI seq info [330701:330701:8:4294967295]
00000400:00000200:3.0:1788575186.325954:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575186.325969:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575186.325980:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575186.325984:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575186.325990:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575186.325992:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575186.325996:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575186.325999:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575186.326002:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575186.326003:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575186.326007:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575186.326010:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575186.326014:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575186.326016:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575186.326019:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575186.326022:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326028:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575186.326031:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326040:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575186.326041:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575186.326045:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575186.326047:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575186.326049:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306790841
00000100:00000010:3.0:1788575186.326053:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575186.326058:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575186.326062:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306790841
00000100:00000010:3.0:1788575186.326064:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788575186.326066:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575186.326068:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575186.326070:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575186.326071:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575186.326074:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575186/real 1788575186]  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575186.326085:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575186.326086:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575186.326087:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575186.326089:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575186.326090:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575186.326091:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575186.326093:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575186.326098:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326102:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575186.326103:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326104:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575186.326105:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575186.326108:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326111:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575186.326114:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326118:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575186.326120:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326121:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575186.326121:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326123:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575186.326124:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575186.326126:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575186.326128:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575186.326129:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575186.326131:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575186.326137:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393190528:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575186.326140:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326141:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575186.326143:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393190528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326146:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575186.326147:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575186.326149:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575186.326151:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575186.326152:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575186.326153:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575186.326154:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575186.326155:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575186.326157:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575186.326158:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575186.326159:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575186.326162:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575186.326162:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326164:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326169:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575186.326170:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575186.326171:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575186.326172:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575186.326173:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575186.326175:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326179:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575186.326180:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575186.326181:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575186.326183:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575186.326184:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575186.326185:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393190656:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575186.326187:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575186.326189:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575186.326190:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575186.326191:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575186.326192:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575186.326193:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575186.326195:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575186.326197:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575186.326199:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575186.326201:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393190656, portal 10
00000100:00000001:3.0:1788575186.326203:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575186.326203:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:3.0:1788575186.326207:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326211:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575186.326212:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575186.326213:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575186.326215:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393190656, offset 0
00000400:00000200:3.0:1788575186.326217:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575186.326219:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575186.326221:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575186.326223:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575186.326224:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575186.326226:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575186.326227:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575186.326229:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330745:330745:256:4294967295] 192.168.202.110@tcp LPNI seq info [330702:330702:8:4294967295]
00000400:00000200:3.0:1788575186.326231:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575186.326235:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575186.326237:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575186.326239:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575186.326241:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575186.326243:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575186.326245:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575186.326247:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575186.326249:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575186.326250:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575186.326252:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575186.326254:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575186.326256:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575186.326257:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575186.326258:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575186.326260:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326265:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575186.326266:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326270:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575186.326271:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575186.326273:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575186.326275:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575186.326277:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575186.326278:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306789926
00000100:00000001:3.0:1788575186.326280:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306790841
00000100:00000001:3.0:1788575186.326281:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306789926
00000100:00000001:3.0:1788575186.326282:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306790841
00000100:00000001:3.0:1788575186.326283:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575186.326284:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575186.326285:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575186.326287:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575186/real 1788575186]  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575186.326291:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575186.326292:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575186.326293:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575186.326295:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575186.326296:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575186.326297:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575186.326299:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575186.326303:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326306:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575186.326307:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326308:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575186.326309:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575186.326310:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326314:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575186.326316:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326319:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575186.326321:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326322:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575186.326322:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326323:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575186.326325:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575186.326326:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575186.326328:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575186.326329:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575186.326331:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575186.326335:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393190656:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575186.326337:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575186.326339:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326343:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575186.326344:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575186.326345:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575186.326346:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575186.326347:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575186.326348:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393190784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575186.326350:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575186.326351:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575186.326351:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575186.326352:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575186.326354:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575186.326356:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575186.326358:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575186.326361:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575186.326363:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575186.326366:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393190784, portal 4
00000100:00000001:3.0:1788575186.326367:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575186.326369:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:3.0:1788575186.326373:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326379:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575186.326381:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575186.326383:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575186.326386:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393190784, offset 0
00000400:00000200:3.0:1788575186.326414:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575186.326417:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575186.326419:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575186.326421:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575186.326423:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575186.326425:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575186.326426:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575186.326428:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330746:330746:256:4294967295] 192.168.202.110@tcp LPNI seq info [330703:330703:8:4294967295]
00000400:00000200:3.0:1788575186.326430:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575186.326434:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575186.326437:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575186.326439:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575186.326442:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575186.326443:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575186.326446:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575186.326447:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575186.326457:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575186.326458:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575186.326460:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575186.326462:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575186.326465:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575186.326467:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575186.326469:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575186.326473:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326480:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575186.326483:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326489:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575186.326490:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575186.326494:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575186.326497:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575186.326500:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575186.326502:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306790073
00000100:00000001:3.0:1788575186.326504:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306790841
00000100:00000001:3.0:1788575186.326506:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306790073
00000100:00000001:3.0:1788575186.326509:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306790841
00000100:00000001:3.0:1788575186.326510:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575186.326511:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575186.326512:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575186.326515:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575186/real 1788575186]  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575186.326520:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575186.326521:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575186.326522:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575186.326524:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575186.326525:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575186.326527:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575186.326529:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575186.326537:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326542:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575186.326544:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326545:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575186.326547:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575186.326550:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326557:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575186.326560:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326567:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575186.326569:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326570:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575186.326571:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326573:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575186.326574:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575186.326575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575186.326577:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575186.326579:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575186.326581:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575186.326586:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393190784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575186.326588:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326589:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575186.326590:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393190656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326594:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575186.326595:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575186.326596:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575186.326598:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575186.326599:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575186.326600:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575186.326601:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575186.326602:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575186.326604:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575186.326604:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575186.326605:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:3.0:1788575186.326607:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575186.326608:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326609:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575186.326610:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393190784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575186.326614:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575186.326614:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575186.326615:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575186.326617:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575186.326618:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575186.326618:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575186.326620:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575186.326620:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575186.326622:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575186.326622:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575186.326623:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:3.0:1788575186.326624:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575186.326624:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326625:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575186.326628:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575186.326628:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575186.326629:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575186.326630:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575186.709476:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575186.709482:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575186.709484:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575186.709486:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575186.709489:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575186.709491:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575186.709492:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575186.709494:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575186.709502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575186.709503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575186.709505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575186.709505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575186.709560:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575186.709564:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575186.709567:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788575187.285470:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306790841, at 4306790848
00000100:00000040:0.0:1788575187.285471:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306790841, at 4306790848
00000100:00000010:1.0:1788575187.285484:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:0.0:1788575187.285487:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575187.733458:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575187.733465:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575187.733471:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575187.733472:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575187.733475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575187.733476:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575188.757443:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575188.757449:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575188.757454:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575188.757456:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575188.757459:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575188.757461:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575189.781533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575189.781539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575189.781544:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575189.781545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575189.781548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575189.781549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575189.781615:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575189.781620:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575189.781622:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00000001:3.0:1788575190.805532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575190.805542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575190.805549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575190.805550:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575190.805555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575190.805556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575191.445571:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575191.445585:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575191.445589:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575191.445594:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575191.445599:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575191.445609:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12122
00000400:00000200:3.0:1788575191.445619:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575191.445626:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575191.445629:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575191.445632:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445635:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575191.445638:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445640:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575191.445645:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575191.445646:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575191.445649:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575191.445653:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575191.445655:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575191.445658:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575191.445660:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445663:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575191.445665:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445667:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575191.445670:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575191.445673:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445676:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575191.445679:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575191.445683:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575191.445687:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575191.445689:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575191.445691:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575191.445702:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575191.445706:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575191.445710:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575191.445712:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575191.445715:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575191.445718:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575191.445722:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575191.445732:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575191.445740:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575191.445752:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393190912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575191.445763:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575191.445768:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575191.445773:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393190912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575191.445795:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575191.445799:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575191.445803:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575191.445805:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575191.445807:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575191.445810:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575191.445814:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12122
00000400:00000200:3.0:1788575191.445818:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575191.445822:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575191.445825:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575191.445829:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445832:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575191.445836:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445838:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575191.445841:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575191.445842:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575191.445844:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575191.445846:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575191.445848:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575191.445850:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575191.445855:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445858:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575191.445861:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445865:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575191.445869:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575191.445872:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445875:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575191.445877:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575191.445883:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575191.445886:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575191.445889:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575191.445895:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575191.445899:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575191.445900:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575191.445904:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000020:00000040:3.0:1788575191.445908:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575191.445912:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575191.445916:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575191.445917:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575191.445918:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575191.445920:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575191.445926:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575191.445931:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575191.445938:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393191040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575191.445944:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575191.445947:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575191.445952:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393191040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575191.445962:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575191.445965:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575191.445968:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575191.445971:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575191.445972:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575191.445974:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575191.445976:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12122
00000400:00000200:3.0:1788575191.445980:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575191.445984:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575191.445985:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575191.445987:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445989:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575191.445992:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.445995:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575191.445998:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575191.446000:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575191.446002:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575191.446004:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575191.446005:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575191.446008:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575191.446012:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.446014:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575191.446016:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575191.446019:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575191.446024:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575191.446027:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575191.446033:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575191.446034:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575191.446038:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575191.446039:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575191.446040:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575191.446042:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575191.446043:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575191.446044:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575191.446046:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788575191.446047:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575191.446051:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575191.446053:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575191.446055:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575191.446056:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575191.446057:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575191.446062:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575191.446065:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575191.446071:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393191168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575191.446081:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575191.446084:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575191.446087:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393191168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575191.446096:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575191.446099:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575191.446101:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575191.446106:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12132)
00000800:00000200:2.1:1788575191.446177:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575191.446188:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575191.446622:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575191.446624:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575191.446864:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575191.446866:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00010000:00000001:3.0:1788575191.829544:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575191.829554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575191.829562:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575191.829564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575191.829568:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575191.829569:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575192.405515:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306795961, at 4306795968
00000100:00000040:1.0:1788575192.405515:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306795961, at 4306795968
00000100:00000010:0.0:1788575192.405532:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000100:00000010:1.0:1788575192.405535:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00010000:00000001:3.0:1788575192.853471:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575192.853479:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575192.853484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575192.853487:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575192.853490:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575192.853493:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575192.853496:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575192.853497:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575192.853499:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575192.853500:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575192.853503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575192.853504:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575192.853505:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575192.853506:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575192.853507:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575192.853508:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575192.853509:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575192.853510:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575192.853574:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575192.853578:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575192.853580:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575192.981570:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575192.981580:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575192.981583:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575192.981589:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575192.981593:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575192.981597:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575192.981600:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575192.981604:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575192.981608:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575193.877500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575193.877507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575193.877512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575193.877512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575193.877515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575193.877516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575194.005510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575194.005520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575194.005522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575194.005528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575194.005532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575194.005537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575194.005540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575194.005544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575194.005547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575194.901494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575194.901503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575194.901508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575194.901511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575194.901514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575194.901515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575195.029503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575195.029508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575195.029510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575195.029514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575195.029516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575195.029519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575195.029520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575195.029523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575195.029524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575195.925484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575195.925492:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575195.925494:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575195.925497:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575195.925501:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575195.925503:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575195.925505:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575195.925507:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575195.925511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575195.925512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575195.925516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575195.925517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575195.925568:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575195.925571:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575195.925573:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575196.053553:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575196.053565:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575196.053568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575196.053575:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575196.053579:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575196.053585:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575196.053588:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575196.053592:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575196.053595:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575196.565545:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575196.565558:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575196.565560:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575196.565564:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575196.565568:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575196.565578:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12127
00000400:00000200:3.0:1788575196.565588:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575196.565594:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575196.565597:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575196.565599:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565602:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575196.565604:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565606:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575196.565610:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575196.565612:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575196.565614:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575196.565617:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575196.565619:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575196.565620:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575196.565623:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565625:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575196.565626:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565628:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575196.565631:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575196.565633:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565635:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575196.565637:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575196.565640:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575196.565642:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575196.565644:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575196.565646:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575196.565656:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575196.565658:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575196.565661:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575196.565663:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575196.565665:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575196.565668:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575196.565672:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575196.565679:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575196.565685:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575196.565693:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393191296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575196.565700:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575196.565704:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575196.565707:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393191296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575196.565724:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575196.565726:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575196.565729:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575196.565730:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575196.565732:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575196.565734:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575196.565736:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12127
00000400:00000200:3.0:1788575196.565738:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575196.565740:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575196.565742:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575196.565745:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565746:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575196.565748:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565749:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575196.565750:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575196.565752:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575196.565753:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575196.565756:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575196.565758:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575196.565761:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575196.565763:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565765:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575196.565766:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565768:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575196.565771:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575196.565775:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565778:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575196.565780:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575196.565786:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575196.565790:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575196.565793:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575196.565800:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575196.565803:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575196.565806:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575196.565810:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000020:00000040:3.0:1788575196.565814:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575196.565818:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575196.565821:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575196.565823:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575196.565825:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575196.565828:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575196.565831:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575196.565834:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575196.565841:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393191424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575196.565848:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575196.565850:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575196.565855:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393191424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575196.565861:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575196.565863:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575196.565865:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575196.565866:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575196.565866:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575196.565869:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575196.565874:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12127
00000400:00000200:3.0:1788575196.565877:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575196.565879:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575196.565883:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575196.565885:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565886:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575196.565887:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565888:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575196.565889:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575196.565891:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575196.565894:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575196.565896:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575196.565897:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575196.565898:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575196.565900:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565902:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575196.565904:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565908:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575196.565909:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575196.565913:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575196.565916:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575196.565919:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575196.565921:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575196.565922:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575196.565925:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575196.565926:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575196.565930:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575196.565933:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575196.565939:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000020:00000040:3.0:1788575196.565943:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575196.565948:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575196.565950:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575196.565953:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575196.565955:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575196.565958:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575196.565964:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788575196.565969:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575196.565976:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393191552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575196.565985:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575196.565987:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575196.565993:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393191552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575196.566000:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575196.566004:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575196.566007:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575196.566012:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12137)
00000800:00000200:2.1:1788575196.566078:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575196.566081:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788575196.566499:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575196.566501:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788575196.566674:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575196.566676:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00010000:00000001:3.0:1788575196.949475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575196.949481:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575196.949486:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575196.949487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575196.949489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575196.949490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575197.077526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575197.077534:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575197.077536:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575197.077542:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575197.077545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575197.077549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575197.077551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575197.077553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575197.077556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575197.525519:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306801081, at 4306801088
00000100:00000040:1.0:1788575197.525519:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306801081, at 4306801088
00000100:00000010:1.0:1788575197.525540:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00000100:00000010:0.0:1788575197.525544:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00010000:00000001:3.0:1788575197.973485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575197.973491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575197.973496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575197.973498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575197.973501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575197.973503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575198.101506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575198.101512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575198.101513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575198.101517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575198.101519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575198.101522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575198.101524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575198.101526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575198.101528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575198.997492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575198.997498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575198.997501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575198.997502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575198.997504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575198.997505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575198.997556:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575198.997559:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575198.997562:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575199.125492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575199.125499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575199.125501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575199.125506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575199.125510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575199.125514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575199.125516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575199.125518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575199.125521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575200.021483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575200.021488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575200.021492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575200.021492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575200.021494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575200.021495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575200.149494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575200.149503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575200.149505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575200.149511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575200.149514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575200.149518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575200.149521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575200.149524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575200.149529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575201.045482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575201.045486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575201.045497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575201.045498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575201.045500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575201.045502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575201.173472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575201.173479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575201.173482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575201.173487:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575201.173491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575201.173495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575201.173498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575201.173501:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575201.173504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575201.685460:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575201.685475:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575201.685478:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575201.685482:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575201.685487:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575201.685494:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12132
00000400:00000200:3.0:1788575201.685500:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575201.685505:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575201.685507:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575201.685509:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685512:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575201.685513:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685515:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575201.685517:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575201.685518:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575201.685520:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575201.685522:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575201.685524:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575201.685526:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575201.685527:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685529:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575201.685530:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685531:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575201.685533:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575201.685535:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685536:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575201.685538:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575201.685541:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575201.685543:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575201.685544:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575201.685546:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575201.685553:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575201.685555:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575201.685558:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575201.685560:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575201.685562:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575201.685564:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575201.685567:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575201.685575:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa000.
00000100:00000001:3.0:1788575201.685581:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575201.685590:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393191680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575201.685600:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575201.685604:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575201.685609:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393191680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575201.685630:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575201.685634:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575201.685637:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575201.685638:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575201.685640:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575201.685643:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575201.685646:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12132
00000400:00000200:3.0:1788575201.685651:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575201.685655:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575201.685658:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575201.685663:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685668:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575201.685673:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685676:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575201.685680:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575201.685683:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575201.685686:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575201.685689:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575201.685693:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575201.685697:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575201.685702:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685707:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575201.685713:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685716:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575201.685720:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575201.685724:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685728:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575201.685733:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575201.685737:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575201.685740:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575201.685741:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575201.685744:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575201.685750:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575201.685751:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575201.685755:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788575201.685759:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575201.685763:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575201.685765:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575201.685766:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575201.685767:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575201.685770:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575201.685775:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
00000100:00000001:3.0:1788575201.685781:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575201.685787:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393191808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575201.685804:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575201.685807:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575201.685814:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393191808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575201.685825:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575201.685827:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575201.685831:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575201.685834:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575201.685835:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575201.685837:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575201.685842:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12132
00000400:00000200:3.0:1788575201.685890:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575201.685895:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575201.685899:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575201.685904:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685906:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575201.685910:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685912:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575201.685914:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575201.685916:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575201.685918:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575201.685919:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575201.685921:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575201.685923:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575201.685930:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685932:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575201.685934:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575201.685939:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575201.685943:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:2.1:1788575201.685944:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1788575201.685949:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:2.1:1788575201.685950:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:3.0:1788575201.685954:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575201.685958:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575201.685962:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575201.685967:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575201.685971:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575201.685974:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575201.685977:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575201.685981:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575201.685985:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000020:00000040:3.0:1788575201.685989:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575201.685995:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575201.685997:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575201.686000:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575201.686002:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575201.686004:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575201.686010:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788575201.686015:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575201.686021:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393191936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575201.686040:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575201.686044:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575201.686048:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393191936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575201.686057:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575201.686061:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575201.686066:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575201.686070:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12142)
00000800:00000200:2.1:1788575201.686266:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575201.686267:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788575201.686539:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575201.686541:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00010000:00000001:3.0:1788575202.069540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575202.069549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575202.069555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575202.069557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575202.069561:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575202.069562:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575202.069639:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575202.069644:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575202.069647:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575202.197542:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575202.197553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575202.197555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575202.197562:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575202.197566:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575202.197570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575202.197573:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575202.197576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575202.197579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575202.645508:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306806201, at 4306806208
00000100:00000040:1.0:1788575202.645508:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306806201, at 4306806208
00000100:00000010:0.0:1788575202.645529:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000100:00000010:1.0:1788575202.645535:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00010000:00000001:3.0:1788575203.093474:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575203.093480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575203.093483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575203.093486:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575203.093487:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575203.093490:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575203.093492:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575203.093493:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575203.093495:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575203.093496:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575203.093498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575203.093499:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575203.093500:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575203.093500:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575203.093501:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575203.093502:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575203.093503:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575203.093504:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575203.221522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575203.221529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575203.221531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575203.221537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575203.221541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575203.221544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575203.221547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575203.221549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575203.221552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575204.117485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575204.117490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575204.117493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575204.117494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575204.117496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575204.117497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575204.245477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575204.245483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575204.245485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575204.245490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575204.245492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575204.245495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575204.245498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575204.245500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575204.245502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575205.141510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575205.141515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575205.141518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575205.141519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575205.141521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575205.141522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575205.141570:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575205.141573:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575205.141575:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575205.269502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575205.269507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575205.269509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575205.269512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575205.269515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575205.269518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575205.269521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575205.269523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575205.269526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575206.165502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575206.165510:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575206.165512:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575206.165516:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575206.165521:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575206.165524:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575206.165526:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575206.165529:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575206.165533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575206.165535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575206.165539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575206.165540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575206.293486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575206.293492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575206.293495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575206.293499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575206.293503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575206.293506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575206.293509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575206.293511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575206.293514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575206.805464:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575206.805475:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575206.805478:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575206.805481:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575206.805486:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575206.805491:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12137
00000400:00000200:3.0:1788575206.805499:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575206.805504:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575206.805506:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575206.805508:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805510:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575206.805512:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805513:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575206.805516:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575206.805517:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575206.805519:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575206.805521:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575206.805522:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575206.805523:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575206.805525:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805526:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575206.805528:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805529:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575206.805531:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575206.805533:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805534:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575206.805537:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575206.805539:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575206.805541:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575206.805543:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575206.805544:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575206.805551:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000020:00000040:3.0:1788575206.805553:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575206.805555:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575206.805556:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575206.805559:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575206.805561:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575206.805564:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575206.805570:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575206.805574:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575206.805581:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393192064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575206.805587:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575206.805590:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575206.805593:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393192064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575206.805610:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575206.805613:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575206.805615:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575206.805616:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575206.805617:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575206.805618:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575206.805620:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12137
00000400:00000200:3.0:1788575206.805622:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575206.805623:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575206.805625:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575206.805626:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805628:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575206.805630:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805631:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575206.805633:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575206.805634:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575206.805635:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575206.805637:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575206.805638:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575206.805640:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575206.805641:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805643:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575206.805647:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805649:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575206.805673:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575206.805679:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805681:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575206.805684:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575206.805689:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575206.805694:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575206.805697:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575206.805701:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575206.805707:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575206.805709:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575206.805713:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1180.
00000020:00000040:3.0:1788575206.805716:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575206.805718:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575206.805719:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575206.805720:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575206.805721:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575206.805722:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575206.805726:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575206.805730:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575206.805738:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1180 x1875444393192192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575206.805749:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575206.805751:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575206.805755:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca1dca9a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1180 x1875444393192192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575206.805761:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575206.805764:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575206.805766:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575206.805767:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575206.805769:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575206.805773:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575206.805778:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12137
00000400:00000200:3.0:1788575206.805783:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575206.805786:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575206.805787:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575206.805791:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805794:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575206.805797:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805800:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575206.805804:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575206.805806:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575206.805809:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575206.805811:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575206.805813:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575206.805817:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575206.805822:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805825:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575206.805827:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805828:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575206.805833:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575206.805835:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575206.805839:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575206.805841:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575206.805843:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575206.805844:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575206.805845:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575206.805847:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575206.805850:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575206.805851:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575206.805856:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000020:00000040:3.0:1788575206.805857:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575206.805861:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575206.805863:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575206.805865:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575206.805866:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575206.805867:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575206.805875:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575206.805878:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575206.805883:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393192320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575206.805897:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575206.805900:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575206.805906:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393192320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575206.805912:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575206.805916:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575206.805921:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575206.805926:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12148)
00000800:00000200:2.1:1788575206.805952:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575206.805955:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575206.806205:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575206.806207:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575206.806378:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575206.806381:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00010000:00000001:3.0:1788575207.189453:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575207.189459:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575207.189462:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575207.189463:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575207.189465:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575207.189466:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575207.765480:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306811321, at 4306811328
00000100:00000040:1.0:1788575207.765480:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306811321, at 4306811328
00000100:00000010:1.0:1788575207.765493:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000100:00000010:0.0:1788575207.765495:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00010000:00000001:3.0:1788575208.213496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575208.213502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575208.213508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575208.213509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575208.213512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575208.213514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575208.213575:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575208.213579:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575208.213582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00000001:3.0:1788575209.237449:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575209.237455:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575209.237458:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575209.237459:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575209.237461:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575209.237462:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575210.261547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575210.261555:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575210.261560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575210.261561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575210.261564:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575210.261565:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575211.285510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575211.285516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575211.285522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575211.285523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575211.285527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575211.285528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575211.285632:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575211.285639:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575211.285644:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.0:1788575211.925512:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788575211.925525:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788575211.925533:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:3.0:1788575211.925683:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575211.925688:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000800:00000001:0.0:1788575211.925691:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000001:3.0:1788575211.925691:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575211.925701:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000001:0.0:1788575211.925703:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788575211.925711:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575211.925716:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.925717:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000800:00000200:1.0:1788575211.925721:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000001:3.0:1788575211.925721:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575211.925728:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575211.925732:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575211.925737:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575211.925744:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393192448:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575211.925751:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575211.925756:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575211.925759:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575211.925761:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575211.925764:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575211.925767:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575211.925773:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575211.925777:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575211.925784:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575211.925791:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393192448, portal 25
00000100:00000001:3.0:1788575211.925794:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575211.925797:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:3.0:1788575211.925806:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.925817:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575211.925820:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575211.925823:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575211.925825:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393192448, offset 0
00000400:00000200:3.0:1788575211.925832:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575211.925844:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575211.925852:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575211.925858:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575211.925864:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575211.925869:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575211.925873:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575211.925880:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330759:330759:256:4294967295] 192.168.202.110@tcp LPNI seq info [330716:330716:8:4294967295]
00000400:00000200:3.0:1788575211.925887:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575211.925901:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575211.925911:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575211.925917:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575211.925924:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575211.925931:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575211.925941:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575211.925949:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575211.925962:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575211.925968:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575211.925975:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575211.925984:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575211.925990:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575211.925993:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575211.925998:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575211.926004:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926017:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575211.926022:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926042:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575211.926044:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575211.926056:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575211.926062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575211.926067:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306816441
00000100:00000010:3.0:1788575211.926072:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575211.926078:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575211.926083:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306816441
00000100:00000010:3.0:1788575211.926085:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575211.926087:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575211.926089:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575211.926091:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575211.926093:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575211.926096:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575211/real 1788575211]  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575211.926108:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575211.926109:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575211.926110:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575211.926112:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575211.926113:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575211.926114:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575211.926120:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575211.926126:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926129:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575211.926130:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926132:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575211.926133:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575211.926136:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926140:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575211.926142:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926146:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575211.926148:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926149:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575211.926149:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926151:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575211.926152:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575211.926154:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575211.926156:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575211.926157:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575211.926159:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575211.926165:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393192448:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575211.926168:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926169:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575211.926171:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393192448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926174:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575211.926176:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575211.926177:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575211.926179:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575211.926181:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575211.926181:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575211.926183:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575211.926184:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575211.926186:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575211.926188:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575211.926189:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575211.926191:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575211.926192:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926193:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926198:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575211.926199:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575211.926200:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575211.926202:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575211.926202:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575211.926204:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926209:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575211.926209:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575211.926211:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575211.926212:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575211.926213:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575211.926215:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393192576:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575211.926217:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575211.926218:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575211.926219:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575211.926220:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575211.926222:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575211.926223:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575211.926225:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575211.926227:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575211.926229:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575211.926231:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393192576, portal 10
00000100:00000001:3.0:1788575211.926232:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575211.926233:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:3.0:1788575211.926237:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926241:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575211.926242:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575211.926244:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575211.926245:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393192576, offset 0
00000400:00000200:3.0:1788575211.926247:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575211.926249:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575211.926252:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575211.926253:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575211.926255:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575211.926257:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575211.926258:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575211.926260:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330760:330760:256:4294967295] 192.168.202.110@tcp LPNI seq info [330717:330717:8:4294967295]
00000400:00000200:3.0:1788575211.926262:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575211.926266:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575211.926269:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575211.926270:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575211.926273:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575211.926275:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575211.926277:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575211.926279:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575211.926281:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575211.926283:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575211.926284:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575211.926286:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575211.926288:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575211.926289:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575211.926291:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575211.926293:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926298:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575211.926299:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926303:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575211.926304:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575211.926306:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575211.926308:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575211.926310:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575211.926312:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306815526
00000100:00000001:3.0:1788575211.926313:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306816441
00000100:00000001:3.0:1788575211.926315:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306815526
00000100:00000001:3.0:1788575211.926316:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306816441
00000100:00000001:3.0:1788575211.926317:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575211.926318:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575211.926319:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575211.926321:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575211/real 1788575211]  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575211.926325:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575211.926326:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575211.926328:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575211.926329:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575211.926330:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575211.926332:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575211.926333:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575211.926338:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926341:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575211.926342:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926343:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575211.926343:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575211.926345:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926349:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575211.926351:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926354:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575211.926356:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926356:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575211.926357:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926358:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575211.926360:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575211.926361:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575211.926363:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575211.926364:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575211.926366:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575211.926371:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393192576:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575211.926373:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575211.926375:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926379:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575211.926379:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575211.926381:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575211.926381:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575211.926383:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575211.926384:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393192704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575211.926386:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575211.926386:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575211.926387:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575211.926388:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575211.926390:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575211.926406:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575211.926408:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575211.926410:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575211.926412:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575211.926414:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393192704, portal 4
00000100:00000001:3.0:1788575211.926415:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575211.926416:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575211.926420:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926424:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575211.926425:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575211.926426:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575211.926427:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393192704, offset 0
00000400:00000200:3.0:1788575211.926430:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575211.926431:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575211.926433:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575211.926435:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575211.926437:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575211.926438:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575211.926454:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575211.926456:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330761:330761:256:4294967295] 192.168.202.110@tcp LPNI seq info [330718:330718:8:4294967295]
00000400:00000200:3.0:1788575211.926459:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575211.926463:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575211.926466:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575211.926468:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575211.926470:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575211.926472:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575211.926474:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575211.926476:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575211.926478:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575211.926479:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575211.926481:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575211.926483:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575211.926485:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575211.926486:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575211.926488:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575211.926490:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926495:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575211.926497:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926500:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575211.926502:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575211.926504:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575211.926505:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575211.926507:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575211.926508:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306815673
00000100:00000001:3.0:1788575211.926510:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306816441
00000100:00000001:3.0:1788575211.926512:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306815673
00000100:00000001:3.0:1788575211.926515:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306816441
00000100:00000001:3.0:1788575211.926517:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575211.926518:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575211.926520:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575211.926524:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575211/real 1788575211]  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575211.926531:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575211.926532:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575211.926534:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575211.926536:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575211.926538:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575211.926540:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575211.926543:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575211.926550:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926557:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575211.926558:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926560:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575211.926562:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575211.926565:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926572:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575211.926576:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926582:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575211.926584:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926586:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575211.926587:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926589:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575211.926598:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575211.926600:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575211.926603:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575211.926605:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575211.926609:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575211.926617:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393192704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575211.926620:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926622:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575211.926625:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393192576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926631:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575211.926633:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575211.926634:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575211.926637:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575211.926638:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575211.926639:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575211.926641:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575211.926643:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575211.926645:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575211.926647:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575211.926648:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575211.926650:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575211.926651:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926653:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575211.926656:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393192704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575211.926662:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575211.926663:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575211.926664:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575211.926667:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575211.926668:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575211.926669:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575211.926671:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575211.926672:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575211.926674:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575211.926675:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575211.926676:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575211.926678:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575211.926679:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926681:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575211.926683:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575211.926684:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575211.926686:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575211.926687:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575212.309483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575212.309489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575212.309495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575212.309496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575212.309500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575212.309502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575212.885506:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306816441, at 4306816448
00000100:00000040:1.0:1788575212.885506:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306816441, at 4306816448
00000100:00000010:0.0:1788575212.885519:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788575212.885526:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575213.333485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575213.333490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575213.333494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575213.333497:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575213.333499:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575213.333501:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575213.333504:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575213.333506:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575213.333507:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575213.333508:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575213.333511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575213.333512:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575213.333513:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575213.333514:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575213.333515:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575213.333516:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575213.333517:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575213.333519:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575214.357459:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575214.357464:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575214.357467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575214.357468:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575214.357471:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575214.357472:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575214.357533:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575214.357537:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575214.357540:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575214.485524:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575214.485529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575214.485531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575214.485534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575214.485537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575214.485540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575214.485542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575214.485543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575214.485545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575215.381496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575215.381502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575215.381505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575215.381506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575215.381508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575215.381509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575215.509460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575215.509466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575215.509467:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575215.509471:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575215.509473:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575215.509476:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575215.509478:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575215.509479:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575215.509481:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575216.405466:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575216.405472:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575216.405474:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575216.405476:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575216.405480:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575216.405481:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575216.405482:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575216.405484:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575216.405486:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575216.405487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575216.405489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575216.405490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575216.533529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575216.533536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575216.533538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575216.533542:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575216.533545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575216.533547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575216.533549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575216.533550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575216.533552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575217.045626:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575217.045630:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575217.045631:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575217.045640:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.045656:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575217.045659:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575217.045664:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575217.045667:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575217.045671:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575217.045677:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393192832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575217.045688:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575217.045692:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575217.045696:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575217.045699:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575217.045704:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575217.045707:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575217.045713:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575217.045717:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575217.045724:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575217.045730:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393192832, portal 25
00000100:00000001:3.0:1788575217.045733:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575217.045736:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:3.0:1788575217.045743:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.045756:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575217.045761:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575217.045764:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575217.045766:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393192832, offset 0
00000400:00000200:3.0:1788575217.045775:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575217.045790:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575217.045796:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575217.045799:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575217.045801:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575217.045804:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575217.045806:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575217.045809:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330762:330762:256:4294967295] 192.168.202.110@tcp LPNI seq info [330719:330719:8:4294967295]
00000400:00000200:3.0:1788575217.045811:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575217.045824:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575217.045834:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575217.045839:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575217.045847:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575217.045854:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788575217.045863:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575217.045867:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1788575217.045875:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575217.045878:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575217.045883:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575217.045889:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575217.045895:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575217.045898:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575217.045904:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575217.045908:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.045919:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575217.045924:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.045934:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575217.045936:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575217.045941:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575217.045947:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575217.045949:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306821561
00000100:00000010:3.0:1788575217.045955:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575217.045961:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575217.045967:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306821561
00000100:00000010:3.0:1788575217.045972:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575217.045977:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575217.045981:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575217.045984:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575217.045987:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575217.045993:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575217/real 1788575217]  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575217.046005:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575217.046008:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575217.046011:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575217.046013:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575217.046015:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575217.046017:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575217.046019:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575217.046025:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046030:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575217.046031:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046032:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575217.046033:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575217.046035:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046039:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575217.046041:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046046:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575217.046047:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046048:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575217.046049:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046051:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575217.046052:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575217.046054:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575217.046056:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575217.046057:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575217.046059:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575217.046063:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393192832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575217.046068:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046069:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575217.046071:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393192832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046075:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575217.046077:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575217.046078:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575217.046080:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575217.046081:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575217.046082:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575217.046083:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575217.046085:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575217.046087:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575217.046088:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575217.046089:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:3.0:1788575217.046091:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575217.046092:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046093:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046098:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575217.046100:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575217.046101:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575217.046102:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575217.046103:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575217.046105:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046109:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575217.046110:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575217.046111:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575217.046112:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575217.046113:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575217.046115:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393192960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575217.046117:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575217.046118:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575217.046119:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575217.046120:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575217.046122:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575217.046123:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575217.046125:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575217.046126:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575217.046128:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575217.046130:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393192960, portal 10
00000100:00000001:3.0:1788575217.046131:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575217.046132:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000040:3.0:1788575217.046136:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046139:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575217.046141:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575217.046142:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575217.046143:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393192960, offset 0
00000400:00000200:3.0:1788575217.046145:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575217.046147:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575217.046149:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575217.046151:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575217.046152:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575217.046154:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575217.046155:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575217.046157:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330763:330763:256:4294967295] 192.168.202.110@tcp LPNI seq info [330720:330720:8:4294967295]
00000400:00000200:3.0:1788575217.046159:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575217.046163:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575217.046166:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575217.046167:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575217.046170:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575217.046171:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788575217.046174:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575217.046176:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1788575217.046178:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575217.046179:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575217.046181:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575217.046183:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575217.046185:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575217.046186:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575217.046187:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575217.046189:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046194:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575217.046195:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046199:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575217.046200:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575217.046202:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575217.046204:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575217.046205:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575217.046206:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306820646
00000100:00000001:3.0:1788575217.046208:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306821561
00000100:00000001:3.0:1788575217.046209:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306820646
00000100:00000001:3.0:1788575217.046211:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306821561
00000100:00000001:3.0:1788575217.046212:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575217.046213:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575217.046214:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575217.046216:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575217/real 1788575217]  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575217.046220:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575217.046221:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575217.046222:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575217.046223:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575217.046224:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575217.046225:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575217.046249:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575217.046255:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046259:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575217.046260:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046261:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575217.046262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575217.046264:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046268:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575217.046270:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046277:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575217.046279:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046280:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575217.046280:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046281:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575217.046283:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575217.046284:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575217.046286:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575217.046288:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575217.046290:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575217.046295:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393192960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575217.046297:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575217.046298:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046306:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575217.046307:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575217.046308:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575217.046309:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575217.046310:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575217.046312:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393193088:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575217.046313:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575217.046314:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575217.046315:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575217.046316:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575217.046318:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575217.046319:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575217.046321:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575217.046323:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575217.046325:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575217.046327:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393193088, portal 4
00000100:00000001:3.0:1788575217.046328:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575217.046328:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575217.046332:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046336:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575217.046337:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575217.046338:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575217.046339:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393193088, offset 0
00000400:00000200:3.0:1788575217.046341:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575217.046343:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575217.046345:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575217.046347:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575217.046348:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575217.046349:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575217.046351:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575217.046353:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330764:330764:256:4294967295] 192.168.202.110@tcp LPNI seq info [330721:330721:8:4294967295]
00000400:00000200:3.0:1788575217.046355:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575217.046359:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575217.046364:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575217.046367:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575217.046370:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575217.046372:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788575217.046376:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575217.046379:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1788575217.046382:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575217.046383:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575217.046386:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575217.046389:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575217.046418:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575217.046420:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575217.046422:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575217.046426:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046434:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575217.046437:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046442:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575217.046443:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575217.046445:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575217.046447:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575217.046450:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575217.046452:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306820793
00000100:00000001:3.0:1788575217.046454:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306821561
00000100:00000001:3.0:1788575217.046456:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306820793
00000100:00000001:3.0:1788575217.046458:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306821561
00000100:00000001:3.0:1788575217.046460:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575217.046470:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575217.046471:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575217.046478:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575217/real 1788575217]  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575217.046486:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575217.046487:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575217.046488:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575217.046491:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575217.046492:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575217.046495:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575217.046498:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575217.046505:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046511:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575217.046512:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046514:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575217.046515:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575217.046519:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046524:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575217.046528:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046534:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575217.046536:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046538:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575217.046539:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046540:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575217.046542:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575217.046545:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575217.046548:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575217.046551:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575217.046555:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575217.046562:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393193088:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575217.046565:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046567:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575217.046570:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393192960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046575:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575217.046576:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575217.046578:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575217.046579:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575217.046580:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575217.046581:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575217.046582:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575217.046583:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575217.046585:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575217.046585:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575217.046586:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:3.0:1788575217.046588:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575217.046589:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046590:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575217.046591:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393193088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575217.046595:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575217.046596:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575217.046597:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575217.046598:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575217.046599:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575217.046599:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575217.046600:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575217.046602:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575217.046603:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575217.046603:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575217.046604:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575217.046606:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575217.046606:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046607:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575217.046609:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575217.046610:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575217.046611:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575217.046612:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575217.429470:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575217.429477:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575217.429483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575217.429484:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575217.429488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575217.429490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575217.429547:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575217.429549:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575217.429552:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575217.557486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575217.557493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575217.557495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575217.557499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575217.557502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575217.557505:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575217.557507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575217.557509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575217.557511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575218.005485:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306821561, at 4306821568
00000100:00000040:1.0:1788575218.005485:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306821561, at 4306821568
00000100:00000010:1.0:1788575218.005499:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:0.0:1788575218.005501:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575218.453510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575218.453515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575218.453519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575218.453519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575218.453522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575218.453523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575218.581543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575218.581550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575218.581552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575218.581558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575218.581562:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575218.581566:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575218.581568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575218.581571:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575218.581574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575219.477495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575219.477501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575219.477504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575219.477505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575219.477508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575219.477509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575219.605563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575219.605572:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575219.605575:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575219.605580:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575219.605584:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575219.605588:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575219.605591:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575219.605594:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575219.605597:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575220.501518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575220.501523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575220.501526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575220.501527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575220.501529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575220.501531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575220.501582:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575220.501586:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575220.501590:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575220.629536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575220.629546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575220.629548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575220.629554:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575220.629559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575220.629564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575220.629567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575220.629570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575220.629574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575221.525507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575221.525512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575221.525515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575221.525517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575221.525519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575221.525520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575221.653568:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575221.653577:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575221.653581:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575221.653587:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575221.653594:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575221.653599:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575221.653602:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575221.653606:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575221.653610:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575222.165711:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575222.165717:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575222.165721:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575222.165729:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.165745:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575222.165748:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575222.165753:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575222.165756:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575222.165760:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575222.165765:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393193216:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575222.165771:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575222.165776:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575222.165779:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575222.165783:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575222.165788:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575222.165791:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575222.165801:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575222.165804:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575222.165813:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575222.165820:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393193216, portal 25
00000100:00000001:3.0:1788575222.165825:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575222.165828:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:3.0:1788575222.165838:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.165855:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575222.165860:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575222.165864:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575222.165868:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393193216, offset 0
00000400:00000200:3.0:1788575222.165881:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575222.165891:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575222.165901:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575222.165910:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575222.165915:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575222.165922:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575222.165927:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575222.165934:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330765:330765:256:4294967295] 192.168.202.110@tcp LPNI seq info [330722:330722:8:4294967295]
00000400:00000200:3.0:1788575222.165952:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575222.165966:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575222.165980:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575222.165990:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575222.166004:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575222.166012:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575222.166036:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575222.166041:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575222.166051:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575222.166055:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575222.166062:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575222.166071:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575222.166080:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575222.166082:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575222.166088:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575222.166093:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166103:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575222.166107:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575222.166116:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575222.166124:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575222.166128:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575222.166131:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306826681
00000100:00000010:3.0:1788575222.166138:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575222.166145:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575222.166152:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306826681
00000100:00000010:3.0:1788575222.166154:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575222.166157:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575222.166159:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575222.166161:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575222.166163:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575222.166166:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575222/real 1788575222]  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575222.166174:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575222.166175:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575222.166177:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575222.166179:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575222.166180:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575222.166183:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575222.166185:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575222.166191:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166195:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575222.166196:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166198:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575222.166199:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575222.166202:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166207:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575222.166210:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166215:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575222.166217:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166219:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575222.166220:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166222:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575222.166223:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575222.166225:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575222.166228:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575222.166229:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575222.166232:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575222.166237:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393193216:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575222.166240:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166242:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575222.166244:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393193216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166249:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575222.166250:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575222.166252:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575222.166254:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575222.166255:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575222.166256:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575222.166258:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575222.166260:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575222.166263:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575222.166264:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575222.166265:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575222.166267:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575222.166268:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166269:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166275:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575222.166276:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575222.166278:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575222.166279:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575222.166280:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575222.166283:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166288:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575222.166289:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575222.166290:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575222.166291:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575222.166293:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575222.166295:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393193344:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575222.166297:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575222.166298:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575222.166299:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575222.166301:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575222.166303:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575222.166304:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575222.166307:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575222.166309:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575222.166311:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575222.166313:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393193344, portal 10
00000100:00000001:3.0:1788575222.166315:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575222.166316:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:3.0:1788575222.166321:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166325:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575222.166326:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575222.166328:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575222.166330:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393193344, offset 0
00000400:00000200:3.0:1788575222.166332:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575222.166335:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575222.166337:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575222.166339:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575222.166341:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575222.166343:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575222.166345:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575222.166347:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330766:330766:256:4294967295] 192.168.202.110@tcp LPNI seq info [330723:330723:8:4294967295]
00000400:00000200:3.0:1788575222.166350:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575222.166355:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575222.166359:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575222.166361:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575222.166364:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575222.166366:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575222.166369:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575222.166371:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575222.166375:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575222.166377:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575222.166380:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575222.166384:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575222.166387:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575222.166389:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575222.166422:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575222.166427:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166434:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575222.166436:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166441:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575222.166442:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575222.166445:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575222.166448:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575222.166451:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575222.166452:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306825767
00000100:00000001:3.0:1788575222.166454:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306826681
00000100:00000001:3.0:1788575222.166456:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306825767
00000100:00000001:3.0:1788575222.166457:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306826681
00000100:00000001:3.0:1788575222.166459:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575222.166461:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575222.166462:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575222.166464:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575222/real 1788575222]  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575222.166470:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575222.166471:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575222.166472:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575222.166473:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575222.166474:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575222.166475:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575222.166477:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575222.166483:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166487:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575222.166488:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166489:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575222.166490:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575222.166492:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166496:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575222.166498:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166503:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575222.166504:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166505:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575222.166506:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166507:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575222.166509:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575222.166511:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575222.166513:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575222.166515:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575222.166517:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575222.166522:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393193344:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575222.166524:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575222.166526:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166530:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575222.166531:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575222.166532:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575222.166534:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575222.166535:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575222.166537:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393193472:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575222.166541:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575222.166543:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575222.166544:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575222.166546:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575222.166550:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575222.166553:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575222.166556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575222.166559:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575222.166563:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575222.166566:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393193472, portal 4
00000100:00000001:3.0:1788575222.166568:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575222.166569:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:3.0:1788575222.166575:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166588:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575222.166591:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575222.166593:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575222.166596:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393193472, offset 0
00000400:00000200:3.0:1788575222.166600:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575222.166605:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575222.166608:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575222.166611:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575222.166614:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575222.166616:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575222.166619:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575222.166622:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330767:330767:256:4294967295] 192.168.202.110@tcp LPNI seq info [330724:330724:8:4294967295]
00000400:00000200:3.0:1788575222.166627:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575222.166634:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575222.166640:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575222.166644:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575222.166648:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575222.166651:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575222.166656:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575222.166659:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575222.166663:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575222.166665:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575222.166669:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575222.166673:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575222.166676:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575222.166678:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575222.166681:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575222.166685:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166694:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575222.166697:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166710:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575222.166712:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575222.166716:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575222.166718:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575222.166721:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575222.166722:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306825913
00000100:00000001:3.0:1788575222.166724:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306826681
00000100:00000001:3.0:1788575222.166725:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306825913
00000100:00000001:3.0:1788575222.166727:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306826681
00000100:00000001:3.0:1788575222.166728:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575222.166730:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575222.166731:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575222.166734:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575222/real 1788575222]  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575222.166739:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575222.166740:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575222.166741:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575222.166743:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575222.166744:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575222.166745:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575222.166748:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575222.166753:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166757:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575222.166758:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166759:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575222.166760:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575222.166762:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166767:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575222.166770:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166774:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575222.166775:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166776:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575222.166777:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166778:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575222.166780:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575222.166781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575222.166783:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575222.166784:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575222.166787:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575222.166791:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393193472:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575222.166794:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166795:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575222.166797:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393193344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166802:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575222.166803:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575222.166804:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575222.166806:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575222.166807:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575222.166808:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575222.166809:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575222.166811:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575222.166813:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575222.166814:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575222.166815:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575222.166817:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575222.166818:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166819:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575222.166821:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393193472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575222.166825:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575222.166826:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575222.166827:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575222.166828:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575222.166829:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575222.166830:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575222.166831:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575222.166832:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575222.166833:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575222.166834:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575222.166834:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:3.0:1788575222.166836:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575222.166836:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166837:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575222.166840:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575222.166841:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575222.166842:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575222.166843:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575222.549523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575222.549530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575222.549536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575222.549537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575222.549541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575222.549542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575222.677609:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575222.677618:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575222.677621:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575222.677629:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575222.677633:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575222.677637:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575222.677640:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575222.677644:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575222.677648:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575223.125534:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306826681, at 4306826688
00000100:00000040:1.0:1788575223.125534:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306826681, at 4306826688
00000100:00000010:1.0:1788575223.125556:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575223.125560:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00010000:00000001:3.0:1788575223.573505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575223.573510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575223.573514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575223.573517:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575223.573519:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575223.573521:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575223.573524:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575223.573525:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575223.573526:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575223.573528:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575223.573530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575223.573531:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575223.573532:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575223.573533:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575223.573534:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575223.573535:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575223.573536:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575223.573537:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575223.573600:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575223.573603:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575223.573606:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575223.701522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575223.701529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575223.701531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575223.701535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575223.701538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575223.701541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575223.701543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575223.701545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575223.701548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575224.597503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575224.597508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575224.597511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575224.597512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575224.597514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575224.597515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575224.725569:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575224.725581:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575224.725584:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575224.725592:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575224.725597:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575224.725602:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575224.725606:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575224.725609:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575224.725613:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575225.621545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575225.621553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575225.621559:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575225.621561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575225.621565:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575225.621566:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575225.749602:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575225.749614:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575225.749617:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575225.749625:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575225.749630:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575225.749634:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575225.749638:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575225.749642:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575225.749645:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575226.645508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575226.645517:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575226.645520:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575226.645523:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575226.645529:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575226.645531:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575226.645532:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575226.645534:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575226.645537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575226.645543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575226.645548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575226.645549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575226.645636:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575226.645641:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575226.645645:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575226.773542:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575226.773552:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575226.773555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575226.773562:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575226.773577:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575226.773586:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575226.773590:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575226.773595:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575226.773599:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575227.285530:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575227.285542:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575227.285545:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575227.285550:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575227.285555:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575227.285562:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12158
00000400:00000200:3.0:1788575227.285572:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575227.285579:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575227.285582:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575227.285584:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285587:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575227.285589:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285592:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575227.285595:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575227.285597:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575227.285600:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575227.285602:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575227.285604:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575227.285606:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575227.285607:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285609:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575227.285610:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285612:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575227.285615:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575227.285617:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285619:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575227.285622:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575227.285625:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575227.285627:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575227.285629:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575227.285632:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575227.285642:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788575227.285646:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575227.285650:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575227.285651:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575227.285654:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575227.285657:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575227.285661:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575227.285669:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575227.285676:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575227.285685:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393193600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575227.285693:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575227.285697:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575227.285701:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393193600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575227.285724:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575227.285726:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575227.285729:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575227.285731:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575227.285732:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575227.285734:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575227.285736:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12158
00000400:00000200:3.0:1788575227.285739:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575227.285741:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575227.285744:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575227.285747:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285748:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575227.285750:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285751:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575227.285754:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575227.285756:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575227.285759:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575227.285762:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575227.285765:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575227.285769:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575227.285773:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285775:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575227.285777:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285781:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575227.285785:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575227.285790:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285796:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575227.285797:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575227.285805:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575227.285822:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575227.285828:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575227.285834:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575227.285838:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575227.285842:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575227.285846:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575227.285849:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575227.285857:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575227.285859:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575227.285861:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575227.285863:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575227.285868:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575227.285873:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575227.285878:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575227.285886:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393193728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575227.285896:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575227.285899:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575227.285903:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393193728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575227.285936:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575227.285940:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575227.285946:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575227.285950:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575227.285952:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575227.285958:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575227.285961:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12158
00000400:00000200:3.0:1788575227.285967:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575227.285969:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575227.285971:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575227.285977:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285980:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575227.285982:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.285984:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575227.285985:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575227.285987:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575227.285990:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575227.285993:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575227.285995:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575227.285997:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575227.286002:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.286007:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575227.286011:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575227.286015:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575227.286027:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575227.286036:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575227.286044:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575227.286048:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575227.286053:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575227.286060:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575227.286065:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575227.286070:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575227.286076:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575227.286081:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575227.286090:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000020:00000040:3.0:1788575227.286096:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.1:1788575227.286102:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:3.0:1788575227.286104:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575227.286107:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575227.286110:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:3.0:1788575227.286111:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575227.286114:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575227.286118:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575227.286123:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575227.286128:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575227.286137:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393193856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575227.286150:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575227.286156:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575227.286162:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393193856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575227.286172:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575227.286175:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575227.286181:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575227.286186:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12168)
00000800:00000200:2.1:1788575227.286600:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575227.286602:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575227.286890:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575227.286893:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00010000:00000001:3.0:1788575227.669522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575227.669531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575227.669539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575227.669541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575227.669545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575227.669547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575227.797533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575227.797542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575227.797545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575227.797551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575227.797556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575227.797559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575227.797562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575227.797565:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575227.797568:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575228.245538:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306831801, at 4306831808
00000100:00000040:1.0:1788575228.245538:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306831801, at 4306831808
00000100:00000010:1.0:1788575228.245559:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000100:00000010:0.0:1788575228.245566:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00010000:00000001:3.0:1788575228.693491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575228.693496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575228.693500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575228.693501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575228.693503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575228.693504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575228.821535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575228.821544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575228.821547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575228.821553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575228.821557:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575228.821561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575228.821564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575228.821566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575228.821569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575229.717547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575229.717552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575229.717556:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575229.717557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575229.717560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575229.717561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575229.717612:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575229.717615:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575229.717618:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575229.845523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575229.845531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575229.845542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575229.845548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575229.845551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575229.845554:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575229.845556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575229.845558:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575229.845560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575230.741452:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575230.741457:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575230.741464:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575230.741465:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575230.741468:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575230.741469:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575230.869529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575230.869538:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575230.869541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575230.869547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575230.869550:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575230.869555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575230.869558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575230.869560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575230.869563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575231.765483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575231.765489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575231.765493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575231.765494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575231.765496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575231.765497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575231.893546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575231.893555:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575231.893558:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575231.893563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575231.893566:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575231.893571:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575231.893573:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575231.893577:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575231.893580:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575232.405718:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575232.405723:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575232.405727:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575232.405734:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.405745:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575232.405748:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575232.405751:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575232.405756:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575232.405761:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575232.405767:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393193984:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575232.405772:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575232.405776:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575232.405778:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575232.405782:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575232.405788:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575232.405792:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575232.405799:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575232.405802:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575232.405810:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575232.405816:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393193984, portal 25
00000100:00000001:3.0:1788575232.405821:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575232.405823:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:3.0:1788575232.405829:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.405840:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575232.405845:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575232.405849:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575232.405854:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393193984, offset 0
00000400:00000200:3.0:1788575232.405863:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575232.405873:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575232.405880:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575232.405886:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575232.405891:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575232.405896:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575232.405900:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575232.405908:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330771:330771:256:4294967295] 192.168.202.110@tcp LPNI seq info [330728:330728:8:4294967295]
00000400:00000200:3.0:1788575232.405912:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575232.405926:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575232.405936:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575232.405942:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575232.405951:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575232.405959:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788575232.405968:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575232.405972:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788575232.405979:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575232.405983:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575232.405990:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575232.405998:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575232.406004:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575232.406007:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575232.406012:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575232.406020:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406034:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575232.406039:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406053:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575232.406056:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575232.406064:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575232.406070:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575232.406075:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306836921
00000100:00000010:3.0:1788575232.406082:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575232.406089:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575232.406097:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306836921
00000100:00000010:3.0:1788575232.406103:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575232.406107:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575232.406111:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575232.406115:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575232.406118:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575232.406125:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575232/real 1788575232]  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575232.406142:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575232.406146:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575232.406150:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575232.406156:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575232.406161:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575232.406166:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575232.406175:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575232.406185:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406192:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575232.406193:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406195:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575232.406197:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575232.406200:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406207:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575232.406212:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406220:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575232.406222:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406224:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575232.406226:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406228:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575232.406230:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575232.406233:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575232.406237:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575232.406240:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575232.406245:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575232.406254:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393193984:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575232.406258:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406260:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575232.406262:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393193984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406272:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575232.406274:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575232.406277:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575232.406281:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575232.406283:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575232.406284:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575232.406287:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575232.406289:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575232.406293:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575232.406295:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575232.406296:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:3.0:1788575232.406300:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575232.406301:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406304:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406311:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575232.406313:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575232.406316:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575232.406317:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575232.406319:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575232.406323:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406331:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575232.406332:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575232.406334:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575232.406336:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575232.406339:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575232.406342:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393194112:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575232.406346:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575232.406348:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575232.406350:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575232.406352:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575232.406356:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575232.406358:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575232.406362:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575232.406365:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575232.406368:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575232.406371:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393194112, portal 10
00000100:00000001:3.0:1788575232.406372:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575232.406373:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:3.0:1788575232.406378:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406383:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575232.406384:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575232.406386:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575232.406387:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393194112, offset 0
00000400:00000200:3.0:1788575232.406390:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575232.406412:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575232.406415:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575232.406418:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575232.406419:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575232.406421:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575232.406423:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575232.406425:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330772:330772:256:4294967295] 192.168.202.110@tcp LPNI seq info [330729:330729:8:4294967295]
00000400:00000200:3.0:1788575232.406427:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575232.406432:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575232.406435:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575232.406438:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575232.406441:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575232.406443:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788575232.406446:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575232.406447:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788575232.406450:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575232.406451:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575232.406453:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575232.406456:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575232.406458:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575232.406459:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575232.406461:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575232.406464:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406469:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575232.406471:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406476:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575232.406477:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575232.406479:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575232.406482:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575232.406484:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575232.406486:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306836007
00000100:00000001:3.0:1788575232.406488:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306836921
00000100:00000001:3.0:1788575232.406489:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306836007
00000100:00000001:3.0:1788575232.406490:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306836921
00000100:00000001:3.0:1788575232.406492:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575232.406493:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575232.406495:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575232.406497:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575232/real 1788575232]  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575232.406502:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575232.406502:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575232.406503:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575232.406505:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575232.406506:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575232.406508:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575232.406510:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575232.406515:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406523:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575232.406524:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406525:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575232.406526:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575232.406528:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406532:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575232.406535:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406539:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575232.406540:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406541:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575232.406542:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406543:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575232.406545:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575232.406546:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575232.406548:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575232.406550:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575232.406552:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575232.406557:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393194112:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575232.406559:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575232.406561:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406566:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575232.406567:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575232.406569:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575232.406570:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575232.406571:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575232.406573:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393194240:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575232.406575:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575232.406577:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575232.406578:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575232.406579:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575232.406581:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575232.406582:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575232.406584:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575232.406586:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575232.406588:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575232.406590:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393194240, portal 4
00000100:00000001:3.0:1788575232.406591:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575232.406592:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:3.0:1788575232.406613:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406618:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575232.406619:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575232.406621:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575232.406623:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393194240, offset 0
00000400:00000200:3.0:1788575232.406625:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575232.406628:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575232.406631:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575232.406633:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575232.406634:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575232.406636:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575232.406638:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575232.406640:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330773:330773:256:4294967295] 192.168.202.110@tcp LPNI seq info [330730:330730:8:4294967295]
00000400:00000200:3.0:1788575232.406643:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575232.406647:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575232.406650:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575232.406652:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575232.406655:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575232.406657:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788575232.406660:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575232.406661:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788575232.406664:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575232.406665:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575232.406667:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575232.406669:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575232.406671:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575232.406672:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575232.406674:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575232.406676:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406685:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575232.406688:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406692:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575232.406693:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575232.406695:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575232.406697:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575232.406699:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575232.406701:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306836153
00000100:00000001:3.0:1788575232.406703:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306836921
00000100:00000001:3.0:1788575232.406704:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306836153
00000100:00000001:3.0:1788575232.406706:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306836921
00000100:00000001:3.0:1788575232.406707:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575232.406708:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575232.406710:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575232.406712:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575232/real 1788575232]  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575232.406717:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575232.406718:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575232.406720:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575232.406721:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575232.406722:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575232.406723:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575232.406725:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575232.406730:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406734:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575232.406735:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406736:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575232.406737:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575232.406739:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406743:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575232.406746:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406750:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575232.406751:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406752:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575232.406753:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406754:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575232.406755:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575232.406756:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575232.406758:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575232.406760:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575232.406763:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575232.406768:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393194240:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575232.406770:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406771:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575232.406773:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393194112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406777:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575232.406778:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575232.406779:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575232.406781:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575232.406782:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575232.406782:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575232.406784:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575232.406785:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575232.406786:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575232.406787:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575232.406788:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575232.406790:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575232.406791:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406792:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575232.406794:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393194240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575232.406798:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575232.406799:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575232.406799:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575232.406801:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575232.406801:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575232.406802:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575232.406803:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575232.406804:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575232.406805:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575232.406806:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575232.406807:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575232.406809:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575232.406809:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406811:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575232.406813:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575232.406814:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575232.406815:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575232.406816:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575232.789499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575232.789504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575232.789508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575232.789510:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575232.789513:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575232.789515:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575232.789517:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575232.789519:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575232.789521:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575232.789522:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575232.789525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575232.789525:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575232.789526:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575232.789527:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575232.789528:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575232.789529:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575232.789530:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575232.789531:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575232.789605:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575232.789608:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575232.789611:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575232.917470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575232.917476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575232.917478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575232.917482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575232.917484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575232.917487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575232.917490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575232.917492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575232.917494:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575233.365472:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306836921, at 4306836928
00000100:00000040:1.0:1788575233.365472:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306836921, at 4306836928
00000100:00000010:1.0:1788575233.365489:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000010:0.0:1788575233.365492:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00010000:00000001:3.0:1788575233.813462:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575233.813468:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575233.813471:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575233.813472:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575233.813474:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575233.813475:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575233.941501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575233.941507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575233.941509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575233.941513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575233.941516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575233.941519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575233.941521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575233.941523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575233.941525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575234.837556:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575234.837564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575234.837570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575234.837571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575234.837576:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575234.837578:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575234.965484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575234.965517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575234.965520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575234.965525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575234.965529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575234.965534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575234.965537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575234.965540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575234.965543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575235.861562:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575235.861575:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575235.861578:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575235.861582:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575235.861589:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575235.861592:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575235.861594:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575235.861597:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575235.861602:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575235.861604:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575235.861609:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575235.861610:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575235.861680:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575235.861684:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575235.861686:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575235.989541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575235.989548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575235.989551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575235.989557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575235.989562:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575235.989566:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575235.989569:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575235.989572:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575235.989575:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575236.885574:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575236.885581:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575236.885593:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575236.885594:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575236.885597:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575236.885598:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575237.013494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575237.013506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575237.013509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575237.013516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575237.013519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575237.013523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575237.013526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575237.013528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575237.013531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575237.525829:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575237.525845:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575237.525850:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575237.525864:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.525881:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575237.525886:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575237.525894:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575237.525900:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575237.525908:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575237.525919:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393194368:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575237.525927:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575237.525936:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575237.525942:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575237.525948:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575237.525958:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575237.525964:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575237.525980:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575237.525989:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575237.526001:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575237.526021:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393194368, portal 25
00000100:00000001:3.0:1788575237.526031:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575237.526035:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:3.0:1788575237.526049:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526079:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575237.526086:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575237.526091:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575237.526096:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393194368, offset 0
00000400:00000200:3.0:1788575237.526104:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575237.526112:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575237.526119:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575237.526124:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575237.526128:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575237.526132:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575237.526135:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575237.526140:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330774:330774:256:4294967295] 192.168.202.110@tcp LPNI seq info [330731:330731:8:4294967295]
00000400:00000200:3.0:1788575237.526146:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575237.526158:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575237.526166:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575237.526173:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575237.526181:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575237.526187:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788575237.526193:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575237.526197:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1788575237.526203:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575237.526206:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575237.526215:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575237.526221:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575237.526228:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575237.526231:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575237.526237:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575237.526242:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526254:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575237.526258:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526267:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575237.526269:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575237.526277:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575237.526283:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575237.526286:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306842041
00000100:00000010:3.0:1788575237.526293:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788575237.526302:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575237.526310:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306842041
00000100:00000010:3.0:1788575237.526314:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575237.526319:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575237.526323:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575237.526327:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575237.526330:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575237.526335:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575237/real 1788575237]  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575237.526348:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575237.526350:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575237.526353:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575237.526356:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575237.526358:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575237.526361:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575237.526365:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575237.526375:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526426:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575237.526429:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526432:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575237.526434:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575237.526440:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526455:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575237.526461:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526470:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575237.526474:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526476:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575237.526478:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526481:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575237.526484:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575237.526487:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575237.526491:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575237.526493:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575237.526497:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575237.526505:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393194368:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575237.526510:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526511:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575237.526514:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393194368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526519:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575237.526521:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575237.526523:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575237.526526:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575237.526527:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575237.526529:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575237.526531:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575237.526533:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575237.526537:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575237.526539:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575237.526540:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:3.0:1788575237.526543:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575237.526544:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526545:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526553:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575237.526555:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575237.526556:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575237.526558:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575237.526559:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575237.526561:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526566:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575237.526567:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575237.526569:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575237.526570:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575237.526572:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575237.526573:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393194496:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575237.526576:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575237.526577:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575237.526579:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575237.526580:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575237.526582:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575237.526584:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575237.526587:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575237.526589:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575237.526592:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575237.526594:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393194496, portal 10
00000100:00000001:3.0:1788575237.526596:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575237.526597:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:3.0:1788575237.526603:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526607:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575237.526609:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575237.526611:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575237.526612:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393194496, offset 0
00000400:00000200:3.0:1788575237.526615:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575237.526618:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575237.526620:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575237.526622:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575237.526624:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575237.526626:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575237.526628:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575237.526630:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330775:330775:256:4294967295] 192.168.202.110@tcp LPNI seq info [330732:330732:8:4294967295]
00000400:00000200:3.0:1788575237.526634:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575237.526638:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575237.526642:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575237.526644:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575237.526647:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575237.526649:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788575237.526652:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575237.526654:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1788575237.526657:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575237.526659:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575237.526661:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575237.526663:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575237.526665:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575237.526667:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575237.526669:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575237.526671:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526681:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575237.526683:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526688:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575237.526689:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575237.526691:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575237.526694:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575237.526696:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575237.526698:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306841127
00000100:00000001:3.0:1788575237.526700:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306842041
00000100:00000001:3.0:1788575237.526701:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306841127
00000100:00000001:3.0:1788575237.526703:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306842041
00000100:00000001:3.0:1788575237.526705:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575237.526706:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575237.526708:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575237.526710:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575237/real 1788575237]  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575237.526715:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575237.526716:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575237.526717:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575237.526718:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575237.526719:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575237.526720:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575237.526722:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575237.526727:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526732:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575237.526733:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526735:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575237.526736:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575237.526743:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526749:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575237.526751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526756:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575237.526757:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526758:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575237.526759:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526760:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575237.526762:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575237.526764:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575237.526766:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575237.526768:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575237.526771:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575237.526776:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393194496:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575237.526779:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575237.526781:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526786:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575237.526786:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575237.526788:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575237.526789:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575237.526790:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575237.526792:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393194624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575237.526795:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575237.526796:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575237.526797:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575237.526799:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575237.526801:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575237.526802:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575237.526804:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575237.526806:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575237.526808:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575237.526810:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393194624, portal 4
00000100:00000001:3.0:1788575237.526811:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575237.526813:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:3.0:1788575237.526817:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526821:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575237.526823:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575237.526824:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575237.526826:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393194624, offset 0
00000400:00000200:3.0:1788575237.526828:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575237.526830:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575237.526832:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575237.526834:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575237.526836:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575237.526837:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575237.526839:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575237.526841:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330776:330776:256:4294967295] 192.168.202.110@tcp LPNI seq info [330733:330733:8:4294967295]
00000400:00000200:3.0:1788575237.526844:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575237.526848:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575237.526851:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575237.526852:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575237.526855:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575237.526857:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788575237.526860:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575237.526862:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1788575237.526865:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575237.526866:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575237.526868:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575237.526871:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575237.526873:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575237.526874:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575237.526875:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575237.526878:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526883:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575237.526885:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526889:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575237.526891:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575237.526893:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575237.526895:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575237.526897:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575237.526898:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306841273
00000100:00000001:3.0:1788575237.526900:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306842041
00000100:00000001:3.0:1788575237.526901:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306841273
00000100:00000001:3.0:1788575237.526903:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306842041
00000100:00000001:3.0:1788575237.526904:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575237.526905:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575237.526907:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575237.526909:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575237/real 1788575237]  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575237.526913:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575237.526914:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575237.526916:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575237.526918:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575237.526920:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575237.526922:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575237.526925:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575237.526935:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526942:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575237.526943:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526945:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575237.526947:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575237.526951:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526959:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575237.526963:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.526970:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575237.526973:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526974:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575237.526975:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.526977:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575237.526979:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575237.526982:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575237.526985:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575237.526988:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575237.526992:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575237.527005:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393194624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575237.527009:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.527011:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575237.527015:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393194496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.527022:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575237.527024:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575237.527026:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575237.527029:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575237.527031:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575237.527032:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575237.527035:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575237.527037:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575237.527040:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575237.527041:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575237.527043:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:3.0:1788575237.527046:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575237.527047:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.527049:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575237.527052:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393194624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575237.527059:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575237.527060:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575237.527062:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575237.527065:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575237.527066:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575237.527068:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575237.527070:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575237.527071:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575237.527073:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575237.527075:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575237.527077:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:3.0:1788575237.527080:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575237.527081:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.527083:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575237.527086:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575237.527088:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575237.527090:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575237.527092:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575237.909519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575237.909526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575237.909532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575237.909533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575237.909537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575237.909539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575238.037512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575238.037518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575238.037519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575238.037524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575238.037528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575238.037532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575238.037535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575238.037537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575238.037540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575238.485506:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306842041, at 4306842048
00000100:00000040:1.0:1788575238.485506:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306842041, at 4306842048
00000100:00000010:0.0:1788575238.485522:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:1.0:1788575238.485525:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575238.933521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575238.933526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575238.933530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575238.933531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575238.933534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575238.933535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575238.933592:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575238.933597:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575238.933601:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575239.061493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575239.061499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575239.061501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575239.061505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575239.061507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575239.061512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575239.061514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575239.061515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575239.061517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575239.957481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575239.957489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575239.957495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575239.957496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575239.957499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575239.957500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575240.085524:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575240.085530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575240.085531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575240.085535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575240.085538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575240.085541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575240.085543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575240.085545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575240.085547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575240.981469:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575240.981475:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575240.981481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575240.981482:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575240.981485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575240.981486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575241.109511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575241.109517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575241.109519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575241.109522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575241.109525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575241.109528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575241.109530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575241.109532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575241.109534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575242.005508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575242.005513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575242.005519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575242.005520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575242.005525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575242.005526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575242.005595:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575242.005598:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575242.005600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575242.133548:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575242.133558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575242.133561:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575242.133567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575242.133572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575242.133577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575242.133580:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575242.133583:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575242.133587:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575242.645553:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575242.645565:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575242.645568:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575242.645573:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575242.645577:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575242.645585:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12173
00000400:00000200:3.0:1788575242.645594:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575242.645601:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575242.645603:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575242.645606:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645609:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575242.645611:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645613:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575242.645617:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575242.645619:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575242.645622:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575242.645625:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575242.645627:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575242.645628:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575242.645630:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645631:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575242.645633:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645634:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575242.645637:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575242.645640:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645642:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575242.645644:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575242.645647:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575242.645650:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575242.645651:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575242.645654:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575242.645665:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000020:00000040:3.0:1788575242.645669:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575242.645673:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575242.645675:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575242.645678:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575242.645681:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575242.645686:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575242.645694:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575242.645701:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575242.645711:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393194752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575242.645720:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575242.645724:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575242.645728:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393194752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575242.645750:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575242.645753:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575242.645756:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575242.645757:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575242.645759:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575242.645761:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575242.645763:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12173
00000400:00000200:3.0:1788575242.645766:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575242.645768:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575242.645770:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575242.645772:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645774:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575242.645776:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645777:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575242.645779:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575242.645780:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575242.645781:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575242.645783:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575242.645785:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575242.645787:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575242.645790:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645791:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575242.645792:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645794:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575242.645795:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575242.645798:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645801:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575242.645803:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575242.645817:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575242.645824:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575242.645829:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575242.645835:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575242.645840:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575242.645842:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575242.645846:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575242.645851:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575242.645855:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575242.645858:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575242.645861:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575242.645864:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575242.645865:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575242.645869:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788575242.645877:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575242.645884:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393194880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575242.645902:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575242.645905:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575242.645909:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393194880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575242.645943:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575242.645950:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575242.645960:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575242.645963:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575242.645967:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575242.645969:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575242.645975:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12173
00000400:00000200:3.0:1788575242.645981:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575242.645984:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575242.645985:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575242.645989:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645992:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575242.645996:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.645998:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575242.646000:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575242.646001:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575242.646005:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575242.646008:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575242.646012:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575242.646014:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575242.646019:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.646022:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575242.646024:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575242.646028:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575242.646032:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575242.646037:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575242.646039:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575242.646044:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575242.646048:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575242.646051:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575242.646055:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575242.646057:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575242.646063:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575242.646067:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575242.646074:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788575242.646078:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575242.646080:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575242.646081:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575242.646083:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575242.646086:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575242.646090:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575242.646096:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:3.0:1788575242.646103:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575242.646110:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393195008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575242.646124:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575242.646128:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575242.646138:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393195008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788575242.646145:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575242.646157:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575242.646160:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575242.646163:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575242.646169:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12183)
00000800:00000200:2.1:1788575242.646471:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575242.646639:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00010000:00000001:3.0:1788575243.029489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575243.029494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575243.029497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575243.029500:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575243.029502:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575243.029505:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575243.029508:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575243.029510:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575243.029511:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575243.029513:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575243.029515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575243.029516:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575243.029517:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575243.029518:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575243.029519:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575243.029520:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575243.029521:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575243.029522:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575243.157536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575243.157545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575243.157548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575243.157555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575243.157559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575243.157563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575243.157565:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575243.157569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575243.157572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575243.605507:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306847161, at 4306847168
00000100:00000040:1.0:1788575243.605507:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306847161, at 4306847168
00000100:00000010:1.0:1788575243.605525:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000100:00000010:0.0:1788575243.605536:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00010000:00000001:3.0:1788575244.053519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575244.053526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575244.053531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575244.053532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575244.053536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575244.053537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575244.181532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575244.181540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575244.181543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575244.181549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575244.181553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575244.181557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575244.181560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575244.181563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575244.181566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575245.077535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575245.077541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575245.077547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575245.077549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575245.077553:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575245.077554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575245.077616:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575245.077621:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575245.077624:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575245.205511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575245.205523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575245.205525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575245.205529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575245.205532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575245.205536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575245.205538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575245.205539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575245.205541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1788575245.717703:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:0.0:1788575245.717749:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1788575245.717765:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575245.717770:0:2350:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:0.0:1788575245.717775:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575245.717779:0:2350:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1788575245.717784:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788575245.717789:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133700.
00000400:00000200:0.0:1788575245.717794:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788575245.717800:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788575245.717807:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c655d8
00000400:00000010:0.0:1788575245.717810:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c655d8.
00000100:00000001:0.0:1788575245.717815:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575245.717825:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e1500 x1875444393194752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575245.717851:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575245.717857:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393194752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575245.717866:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575245.717868:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788575245.717879:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788575245.717882:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133e00.
00000400:00000200:0.0:1788575245.717887:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788575245.717892:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788575245.717896:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65a18
00000400:00000010:0.0:1788575245.717899:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65a18.
00000100:00000001:0.0:1788575245.717905:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575245.717914:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e1f80 x1875444393194880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575245.717928:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575245.717936:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393194880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575245.717947:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575245.717952:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788575245.717962:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788575245.717968:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:0.0:1788575245.717975:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788575245.717986:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788575245.717994:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c652a8
00000400:00000010:0.0:1788575245.717999:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c652a8.
00000100:00000001:0.0:1788575245.718007:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575245.718015:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e3800 x1875444393195008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575245.718038:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575245.718110:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393195008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575245.718127:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575245.718131:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00010000:00000001:3.0:1788575246.101490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575246.101497:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575246.101499:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575246.101502:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575246.101506:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575246.101508:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575246.101510:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575246.101511:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575246.101514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575246.101522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575246.101526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575246.101528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575246.229485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575246.229492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575246.229494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575246.229498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575246.229501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575246.229504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575246.229506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575246.229508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575246.229510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575247.125548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575247.125555:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575247.125560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575247.125561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575247.125564:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575247.125565:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575247.253482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575247.253490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575247.253493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575247.253498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575247.253502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575247.253506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575247.253509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575247.253511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575247.253515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575247.765705:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575247.765712:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575247.765717:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575247.765728:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.765756:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575247.765760:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575247.765765:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575247.765770:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575247.765776:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575247.765785:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393195136:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575247.765794:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575247.765800:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575247.765805:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575247.765810:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575247.765819:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575247.765824:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575247.765834:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575247.765841:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575247.765850:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575247.765860:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393195136, portal 25
00000100:00000001:3.0:1788575247.765866:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575247.765870:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:3.0:1788575247.765883:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.765903:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575247.765910:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575247.765917:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575247.765923:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393195136, offset 0
00000400:00000200:3.0:1788575247.765937:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575247.765954:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575247.765995:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575247.766005:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575247.766013:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575247.766023:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575247.766041:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575247.766049:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330780:330780:256:4294967295] 192.168.202.110@tcp LPNI seq info [330737:330737:8:4294967295]
00000400:00000200:3.0:1788575247.766060:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575247.766080:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575247.766091:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575247.766102:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575247.766114:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575247.766122:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1788575247.766133:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575247.766140:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000100:3.0:1788575247.766151:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575247.766157:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575247.766165:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575247.766175:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575247.766183:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575247.766186:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575247.766191:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575247.766197:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766208:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575247.766212:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766221:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575247.766222:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575247.766230:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575247.766234:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575247.766237:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306852281
00000100:00000010:3.0:1788575247.766243:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575247.766251:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575247.766256:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306852281
00000100:00000010:3.0:1788575247.766259:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575247.766263:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575247.766266:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575247.766269:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575247.766272:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575247.766277:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575247/real 1788575247]  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575247.766288:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575247.766289:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575247.766292:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575247.766295:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575247.766297:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575247.766300:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575247.766303:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575247.766312:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766318:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575247.766320:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.766323:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575247.766325:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575247.766329:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766337:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575247.766343:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766350:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575247.766354:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.766356:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575247.766358:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.766360:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575247.766363:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575247.766366:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575247.766371:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575247.766374:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575247.766378:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575247.766387:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393195136:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575247.766521:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.766524:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575247.766529:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393195136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766536:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575247.766539:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575247.766542:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575247.766545:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575247.766547:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575247.766548:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575247.766551:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575247.766554:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575247.766557:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575247.766559:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575247.766561:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:3.0:1788575247.766565:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575247.766567:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.766569:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.766577:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575247.766579:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575247.766582:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575247.766583:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575247.766585:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575247.766590:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766597:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575247.766599:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575247.766601:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575247.766603:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575247.766606:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575247.766609:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393195264:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575247.766613:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575247.766615:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575247.766617:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575247.766619:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575247.766623:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575247.766625:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575247.766629:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575247.766632:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575247.766636:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575247.766646:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393195264, portal 10
00000100:00000001:3.0:1788575247.766648:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575247.766649:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:3.0:1788575247.766657:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766664:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575247.766666:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575247.766669:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575247.766672:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393195264, offset 0
00000400:00000200:3.0:1788575247.766677:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575247.766682:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575247.766687:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575247.766691:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575247.766693:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575247.766696:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575247.766699:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575247.766702:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330781:330781:256:4294967295] 192.168.202.110@tcp LPNI seq info [330738:330738:8:4294967295]
00000400:00000200:3.0:1788575247.766707:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575247.766715:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575247.766720:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575247.766724:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575247.766729:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575247.766732:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1788575247.766736:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575247.766743:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000100:3.0:1788575247.766746:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575247.766748:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575247.766751:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575247.766755:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575247.766758:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575247.766761:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575247.766763:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575247.766768:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766776:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575247.766780:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766787:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575247.766788:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575247.766792:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575247.766796:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575247.766799:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575247.766801:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306851367
00000100:00000001:3.0:1788575247.766805:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306852281
00000100:00000001:3.0:1788575247.766807:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306851367
00000100:00000001:3.0:1788575247.766809:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306852281
00000100:00000001:3.0:1788575247.766812:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575247.766814:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575247.766816:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575247.766821:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575247/real 1788575247]  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575247.766829:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575247.766831:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575247.766833:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575247.766835:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575247.766837:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575247.766839:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575247.766842:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575247.766849:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766855:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575247.766857:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.766859:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575247.766860:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575247.766863:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766870:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575247.766873:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766880:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575247.766882:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.766884:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575247.766885:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.766887:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575247.766889:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575247.766893:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575247.766896:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575247.766898:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575247.766902:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575247.766909:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393195264:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575247.766914:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575247.766918:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766924:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575247.766925:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575247.766927:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575247.766929:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575247.766931:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575247.766934:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393195392:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575247.766936:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575247.766938:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575247.766940:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575247.766942:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575247.766946:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575247.766948:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575247.766953:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575247.766956:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575247.766959:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575247.766975:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393195392, portal 4
00000100:00000001:3.0:1788575247.766977:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575247.766978:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:3.0:1788575247.766984:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.766997:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575247.766999:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575247.767002:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575247.767005:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393195392, offset 0
00000400:00000200:3.0:1788575247.767009:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575247.767013:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575247.767017:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575247.767020:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575247.767022:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575247.767024:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575247.767027:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575247.767030:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330782:330782:256:4294967295] 192.168.202.110@tcp LPNI seq info [330739:330739:8:4294967295]
00000400:00000200:3.0:1788575247.767034:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575247.767040:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575247.767045:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575247.767049:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575247.767052:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575247.767055:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1788575247.767060:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575247.767063:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000100:3.0:1788575247.767066:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575247.767068:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575247.767071:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575247.767075:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575247.767077:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575247.767079:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575247.767081:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575247.767086:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.767093:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575247.767097:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.767110:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575247.767112:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575247.767116:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575247.767119:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575247.767123:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575247.767125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306851513
00000100:00000001:3.0:1788575247.767128:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306852281
00000100:00000001:3.0:1788575247.767131:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306851513
00000100:00000001:3.0:1788575247.767133:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306852281
00000100:00000001:3.0:1788575247.767135:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575247.767137:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575247.767140:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575247.767144:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575247/real 1788575247]  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575247.767151:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575247.767153:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575247.767155:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575247.767157:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575247.767159:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575247.767161:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575247.767164:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575247.767172:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.767178:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575247.767179:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.767181:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575247.767182:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575247.767186:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.767193:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575247.767197:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.767205:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575247.767207:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.767209:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575247.767210:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.767212:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575247.767214:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575247.767217:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575247.767221:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575247.767223:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575247.767227:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575247.767235:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393195392:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575247.767239:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.767248:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575247.767254:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393195264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.767261:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575247.767262:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575247.767265:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575247.767268:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575247.767270:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575247.767271:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575247.767273:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575247.767275:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575247.767279:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575247.767280:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575247.767282:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:3.0:1788575247.767284:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575247.767285:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.767287:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575247.767290:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393195392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575247.767297:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575247.767298:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575247.767300:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575247.767302:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575247.767303:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575247.767304:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575247.767306:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575247.767307:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575247.767309:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575247.767310:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575247.767311:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:3.0:1788575247.767314:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575247.767315:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.767317:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575247.767322:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575247.767323:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575247.767325:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575247.767327:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575248.149515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575248.149529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575248.149534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575248.149536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575248.149540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575248.149542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575248.149596:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575248.149600:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575248.149604:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575248.277494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575248.277502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575248.277504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575248.277509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575248.277511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575248.277515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575248.277517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575248.277519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575248.277522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575248.725488:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306852281, at 4306852288
00000100:00000040:0.0:1788575248.725489:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306852281, at 4306852288
00000100:00000010:1.0:1788575248.725504:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575248.725507:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00010000:00000001:3.0:1788575249.173480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575249.173485:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575249.173489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575249.173490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575249.173492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575249.173493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575249.301505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575249.301512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575249.301513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575249.301518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575249.301521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575249.301524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575249.301526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575249.301528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575249.301530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788575249.813510:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575249.813517:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788575249.813522:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575249.813681:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575249.813692:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575249.813694:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575249.813700:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575249.813703:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575250.197539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575250.197547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575250.197551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575250.197553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575250.197555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575250.197557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575250.325535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575250.325548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575250.325551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575250.325560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575250.325565:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575250.325570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575250.325575:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575250.325579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575250.325583:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575251.221512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575251.221521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575251.221527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575251.221528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575251.221533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575251.221535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575251.221604:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575251.221609:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575251.221614:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575251.349558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575251.349569:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575251.349571:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575251.349578:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575251.349581:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575251.349586:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575251.349588:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575251.349591:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575251.349594:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575252.245513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575252.245520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575252.245525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575252.245527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575252.245530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575252.245532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575252.373487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575252.373495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575252.373497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575252.373504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575252.373507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575252.373512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575252.373514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575252.373517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575252.373520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575252.885933:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575252.885937:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575252.885940:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575252.885946:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.885955:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575252.885957:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575252.885960:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575252.885963:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575252.885966:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575252.885970:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393195520:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575252.885974:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575252.885979:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575252.885981:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575252.885983:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575252.885988:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575252.885990:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575252.885996:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575252.886000:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575252.886034:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575252.886042:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393195520, portal 25
00000100:00000001:3.0:1788575252.886045:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575252.886047:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530777728 : -130484178773888 : ffff895348e01880)
00000100:00000040:3.0:1788575252.886053:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.886061:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575252.886065:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575252.886068:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575252.886072:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393195520, offset 0
00000400:00000200:3.0:1788575252.886079:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575252.886094:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575252.886648:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575252.886655:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575252.886661:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575252.886666:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575252.886669:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575252.886674:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330783:330783:256:4294967295] 192.168.202.110@tcp LPNI seq info [330740:330740:8:4294967295]
00000400:00000200:3.0:1788575252.886681:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575252.886696:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575252.886702:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575252.886709:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575252.886717:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575252.886722:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1788575252.886728:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575252.886731:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:3.0:1788575252.886736:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575252.886739:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575252.886745:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575252.886762:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575252.886768:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575252.886772:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575252.886776:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575252.886779:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.886787:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575252.886790:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.886795:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575252.886796:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575252.886801:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575252.886805:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575252.886807:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306857402
00000100:00000010:3.0:1788575252.886812:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575252.886817:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575252.886822:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306857402
00000100:00000010:3.0:1788575252.886823:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575252.886826:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575252.886828:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575252.886830:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575252.886832:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575252.886836:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575252/real 1788575252]  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575252.886843:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575252.886844:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575252.886846:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575252.886848:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575252.886849:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575252.886851:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575252.886853:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575252.886859:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.886864:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575252.886865:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.886867:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575252.886868:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575252.886871:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.886875:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575252.886878:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.886883:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575252.886885:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.886886:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575252.886886:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.886888:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575252.886890:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575252.886891:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575252.886894:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575252.886895:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575252.886898:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575252.886903:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393195520:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575252.886907:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.886909:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575252.886911:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01880 x1875444393195520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.886915:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575252.886916:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575252.886919:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575252.886921:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575252.886922:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575252.886923:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575252.886925:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575252.886927:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575252.886930:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575252.886931:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575252.886932:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01880.
00000100:00000001:3.0:1788575252.886935:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575252.886935:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.886937:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.886943:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575252.886944:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575252.886945:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575252.886947:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575252.886948:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575252.886950:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.886955:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575252.886956:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575252.886957:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575252.886959:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575252.886960:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575252.886962:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393195648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575252.886964:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575252.886965:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575252.886966:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575252.886968:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575252.886970:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575252.886971:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575252.886973:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575252.886975:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575252.886977:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575252.886979:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393195648, portal 10
00000100:00000001:3.0:1788575252.886981:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575252.886982:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:3.0:1788575252.886987:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.886991:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575252.886993:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575252.886994:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575252.886996:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393195648, offset 0
00000400:00000200:3.0:1788575252.886998:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575252.887001:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575252.887004:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575252.887006:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575252.887008:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575252.887010:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575252.887011:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575252.887013:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330784:330784:256:4294967295] 192.168.202.110@tcp LPNI seq info [330741:330741:8:4294967295]
00000400:00000200:3.0:1788575252.887016:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575252.887021:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575252.887024:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575252.887026:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575252.887029:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575252.887031:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1788575252.887034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575252.887036:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:3.0:1788575252.887039:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575252.887040:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575252.887042:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575252.887045:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575252.887047:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575252.887048:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575252.887049:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575252.887051:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887061:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575252.887063:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887067:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575252.887068:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575252.887070:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575252.887072:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575252.887074:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575252.887075:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306856487
00000100:00000001:3.0:1788575252.887077:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306857402
00000100:00000001:3.0:1788575252.887079:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306856487
00000100:00000001:3.0:1788575252.887080:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306857402
00000100:00000001:3.0:1788575252.887082:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575252.887083:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575252.887084:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575252.887086:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575252/real 1788575252]  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575252.887091:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575252.887092:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575252.887093:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575252.887094:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575252.887095:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575252.887096:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575252.887098:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575252.887103:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887108:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575252.887109:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887110:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575252.887111:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575252.887114:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887118:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575252.887120:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887124:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575252.887125:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887126:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575252.887127:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887128:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575252.887130:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575252.887131:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575252.887133:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575252.887134:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575252.887137:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575252.887141:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393195648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575252.887144:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575252.887146:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887150:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575252.887151:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575252.887152:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575252.887153:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575252.887155:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575252.887157:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393195776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575252.887159:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575252.887160:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575252.887162:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575252.887163:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575252.887165:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575252.887166:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575252.887168:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575252.887169:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575252.887171:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575252.887173:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393195776, portal 4
00000100:00000001:3.0:1788575252.887174:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575252.887176:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:3.0:1788575252.887179:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887183:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575252.887185:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575252.887186:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575252.887187:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393195776, offset 0
00000400:00000200:3.0:1788575252.887190:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575252.887192:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575252.887193:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575252.887195:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575252.887197:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575252.887198:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575252.887200:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575252.887202:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330785:330785:256:4294967295] 192.168.202.110@tcp LPNI seq info [330742:330742:8:4294967295]
00000400:00000200:3.0:1788575252.887205:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575252.887208:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575252.887211:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575252.887213:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575252.887215:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575252.887217:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1788575252.887220:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575252.887222:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:3.0:1788575252.887224:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575252.887226:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575252.887227:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575252.887230:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575252.887231:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575252.887232:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575252.887234:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575252.887236:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887242:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575252.887245:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887252:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575252.887254:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575252.887257:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575252.887259:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575252.887262:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575252.887263:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306856633
00000100:00000001:3.0:1788575252.887265:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306857402
00000100:00000001:3.0:1788575252.887266:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306856633
00000100:00000001:3.0:1788575252.887267:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306857402
00000100:00000001:3.0:1788575252.887269:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575252.887270:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575252.887271:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575252.887273:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575252/real 1788575252]  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575252.887278:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575252.887279:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575252.887281:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575252.887282:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575252.887283:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575252.887284:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575252.887287:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575252.887292:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887295:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575252.887296:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887297:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575252.887298:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575252.887300:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887304:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575252.887307:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887315:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575252.887317:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887318:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575252.887319:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887320:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575252.887321:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575252.887323:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575252.887325:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575252.887326:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575252.887330:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575252.887341:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393195776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575252.887343:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887345:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575252.887347:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393195648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887353:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575252.887355:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575252.887357:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575252.887359:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575252.887361:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575252.887362:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575252.887364:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575252.887366:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575252.887377:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575252.887379:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575252.887381:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:3.0:1788575252.887384:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575252.887385:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887387:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575252.887408:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393195776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575252.887413:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575252.887415:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575252.887416:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575252.887418:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575252.887419:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575252.887420:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575252.887421:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575252.887422:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575252.887424:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575252.887424:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575252.887426:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:3.0:1788575252.887428:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575252.887429:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887430:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575252.887438:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575252.887439:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575252.887440:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575252.887441:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575253.269467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575253.269480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575253.269487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575253.269490:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575253.269493:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575253.269495:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575253.269498:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575253.269500:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575253.269502:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575253.269503:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575253.269506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575253.269507:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575253.269507:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575253.269508:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575253.269509:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575253.269510:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575253.269511:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575253.269513:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575253.397493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575253.397501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575253.397503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575253.397509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575253.397512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575253.397517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575253.397520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575253.397522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575253.397526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575253.845496:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306857402, at 4306857408
00000100:00000040:1.0:1788575253.845496:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306857402, at 4306857408
00000100:00000010:0.0:1788575253.845511:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788575253.845514:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575254.293503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575254.293509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575254.293516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575254.293517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575254.293521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575254.293522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575254.293583:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575254.293586:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575254.293589:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575254.421501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575254.421508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575254.421510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575254.421524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575254.421532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575254.421536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575254.421539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575254.421541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575254.421545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575255.317533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575255.317539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575255.317543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575255.317544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575255.317546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575255.317547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575256.341495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575256.341502:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575256.341504:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575256.341506:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575256.341509:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575256.341511:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575256.341513:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575256.341514:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575256.341517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575256.341518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575256.341520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575256.341521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575257.365484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575257.365506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575257.365511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575257.365512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575257.365515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575257.365517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575257.365578:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575257.365583:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575257.365586:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1788575258.005494:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575258.005502:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575258.005505:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575258.005508:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575258.005512:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575258.005517:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12189
00000400:00000200:0.0:1788575258.005524:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575258.005529:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575258.005532:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575258.005534:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005536:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575258.005563:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005566:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575258.005569:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575258.005570:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575258.005573:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575258.005575:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575258.005577:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575258.005580:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575258.005582:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005583:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575258.005585:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005586:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575258.005589:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575258.005591:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005593:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575258.005595:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575258.005598:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575258.005600:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575258.005601:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575258.005603:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575258.005616:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788575258.005619:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575258.005622:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575258.005623:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575258.005625:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575258.005628:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575258.005631:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575258.005637:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788575258.005643:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575258.005650:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575258.005658:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575258.005662:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575258.005666:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575258.005681:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575258.005684:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575258.005687:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575258.005688:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575258.005689:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575258.005692:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575258.005694:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12189
00000400:00000200:0.0:1788575258.005697:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575258.005700:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575258.005701:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575258.005704:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005706:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575258.005708:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005709:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575258.005711:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575258.005711:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575258.005712:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575258.005716:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1788575258.005718:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575258.005721:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788575258.005722:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575258.005725:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575258.005727:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575258.005728:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005731:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575258.005733:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005735:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575258.005738:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:3.0:1788575258.005738:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788575258.005744:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005748:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575258.005752:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575258.005756:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788575258.005759:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575258.005759:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575258.005764:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575258.005765:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575258.005767:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575258.005769:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575258.005771:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1788575258.005776:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00100000:3.0:1788575258.005776:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393195904:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575258.005780:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:0.0:1788575258.005783:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575258.005784:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575258.005787:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575258.005789:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575258.005790:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:0.0:1788575258.005797:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
02000000:00000010:3.0:1788575258.005797:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575258.005802:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575258.005803:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575258.005809:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1788575258.005809:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575258.005812:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575258.005813:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000001:0.0:1788575258.005814:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575258.005816:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575258.005819:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575258.005821:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
02000000:00000010:0.0:1788575258.005824:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000200:3.0:1788575258.005828:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393195904, portal 25
00000100:00000001:0.0:1788575258.005829:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.005835:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575258.005838:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00100000:0.0:1788575258.005839:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1788575258.005846:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575258.005851:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575258.005857:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.005857:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575258.005862:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575258.005865:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000040:0.0:1788575258.005867:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575258.005869:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393195904, offset 0
00000400:00000200:3.0:1788575258.005880:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005888:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575258.005892:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788575258.005894:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575258.005899:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:0.0:1788575258.005900:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575258.005903:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575258.005904:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575258.005905:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575258.005908:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:3.0:1788575258.005910:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1788575258.005913:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12189
00000400:00000200:3.0:1788575258.005916:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575258.005918:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575258.005922:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575258.005925:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575258.005929:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575258.005931:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330786:330786:256:4294967295] 192.168.202.110@tcp LPNI seq info [330743:330743:8:4294967295]
00000100:00000040:0.0:1788575258.005935:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005938:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575258.005940:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005942:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575258.005943:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575258.005945:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575258.005948:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1788575258.005948:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1788575258.005952:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575258.005955:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575258.005957:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575258.005963:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575258.005966:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788575258.005967:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575258.005972:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575258.005974:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575258.005979:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:3.0:1788575258.005979:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788575258.005986:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000010:3.0:1788575258.005987:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00000001:0.0:1788575258.005988:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575258.005992:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575258.005995:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575258.005998:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575258.005999:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575258.006001:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575258.006004:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:3.0:1788575258.006004:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000008:00000001:0.0:1788575258.006009:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575258.006012:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575258.006013:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:0.0:1788575258.006018:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000800:00000200:3.1:1788575258.006018:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000040:0.0:1788575258.006023:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000100:3.0:1788575258.006028:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575258.006029:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:3.0:1788575258.006032:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575258.006038:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1788575258.006039:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575258.006045:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1788575258.006045:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1788575258.006048:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575258.006051:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
02000000:00000001:0.0:1788575258.006052:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575258.006055:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
02000000:00000010:0.0:1788575258.006058:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:3.0:1788575258.006060:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:0.0:1788575258.006063:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575258.006066:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575258.006071:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575258.006081:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1788575258.006083:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575258.006087:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575258.006088:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575258.006093:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575258.006101:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575258.006107:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575258.006107:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.006112:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:0.0:1788575258.006113:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575258.006120:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12199)
00000100:00000001:3.0:1788575258.006124:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575258.006130:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575258.006133:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306862521
00000100:00000010:3.0:1788575258.006140:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575258.006147:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575258.006154:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306862521
00000100:00000010:3.0:1788575258.006159:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575258.006162:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575258.006166:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575258.006168:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.006171:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575258.006175:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575258/real 1788575258]  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575258.006183:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575258.006184:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575258.006186:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575258.006188:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575258.006189:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575258.006191:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575258.006193:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575258.006199:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006203:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575258.006205:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006207:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575258.006209:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575258.006212:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006216:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575258.006219:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006224:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575258.006226:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006227:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575258.006227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006229:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575258.006231:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575258.006232:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575258.006235:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575258.006237:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575258.006240:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575258.006246:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393195904:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575258.006249:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006250:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575258.006252:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393195904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006256:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575258.006259:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575258.006260:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575258.006263:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575258.006264:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575258.006266:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575258.006268:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575258.006270:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:3.0:1788575258.006273:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575258.006274:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575258.006275:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:3.0:1788575258.006277:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575258.006278:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006280:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006286:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575258.006287:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.006289:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575258.006290:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575258.006291:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575258.006293:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006298:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575258.006299:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575258.006301:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575258.006302:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575258.006304:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575258.006305:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393196032:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575258.006307:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575258.006309:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575258.006310:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575258.006311:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575258.006314:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575258.006315:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575258.006317:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575258.006328:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575258.006333:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575258.006336:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393196032, portal 10
00000100:00000001:3.0:1788575258.006337:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575258.006338:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000040:3.0:1788575258.006388:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006411:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575258.006413:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575258.006416:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575258.006418:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393196032, offset 0
00000400:00000200:3.0:1788575258.006420:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575258.006423:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575258.006426:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575258.006429:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575258.006430:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575258.006432:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575258.006434:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575258.006436:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330787:330787:256:4294967295] 192.168.202.110@tcp LPNI seq info [330744:330744:8:4294967295]
00000400:00000200:3.0:1788575258.006439:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575258.006443:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575258.006446:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575258.006449:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575258.006451:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575258.006453:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1788575258.006457:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575258.006458:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1788575258.006461:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575258.006462:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575258.006464:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575258.006467:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575258.006469:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575258.006470:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575258.006471:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575258.006474:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006479:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575258.006482:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006495:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.006497:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575258.006501:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575258.006505:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575258.006509:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575258.006511:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306861607
00000100:00000001:3.0:1788575258.006514:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306862521
00000100:00000001:3.0:1788575258.006516:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306861607
00000100:00000001:3.0:1788575258.006518:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306862521
00000100:00000001:3.0:1788575258.006520:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575258.006522:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.006525:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575258.006529:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575258/real 1788575258]  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575258.006551:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575258.006553:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575258.006555:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575258.006558:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575258.006559:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575258.006562:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575258.006566:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575258.006575:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006583:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575258.006584:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006586:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575258.006588:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575258.006592:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006600:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575258.006604:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006609:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575258.006611:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006612:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575258.006613:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006614:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575258.006615:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575258.006617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575258.006619:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575258.006620:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575258.006623:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575258.006628:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393196032:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575258.006631:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575258.006633:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006637:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575258.006638:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575258.006639:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575258.006641:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575258.006642:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575258.006644:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393196160:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575258.006646:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575258.006648:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575258.006649:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575258.006650:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575258.006653:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575258.006655:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575258.006658:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575258.006660:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575258.006662:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575258.006664:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393196160, portal 4
00000100:00000001:3.0:1788575258.006665:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575258.006666:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:3.0:1788575258.006670:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006675:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575258.006676:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575258.006678:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575258.006679:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393196160, offset 0
00000400:00000200:3.0:1788575258.006682:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575258.006685:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575258.006687:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575258.006689:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575258.006691:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575258.006693:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575258.006695:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575258.006697:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330788:330788:256:4294967295] 192.168.202.110@tcp LPNI seq info [330745:330745:8:4294967295]
00000400:00000200:3.0:1788575258.006700:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575258.006704:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575258.006707:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575258.006709:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575258.006712:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575258.006713:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1788575258.006716:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575258.006718:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1788575258.006721:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575258.006723:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575258.006725:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575258.006727:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575258.006729:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575258.006730:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575258.006732:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575258.006734:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006739:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575258.006741:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006746:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.006747:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575258.006749:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575258.006751:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575258.006753:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575258.006754:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306861753
00000100:00000001:3.0:1788575258.006756:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306862521
00000100:00000001:3.0:1788575258.006758:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306861753
00000100:00000001:3.0:1788575258.006759:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306862521
00000100:00000001:3.0:1788575258.006761:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575258.006762:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.006763:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575258.006766:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575258/real 1788575258]  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575258.006770:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575258.006771:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575258.006772:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575258.006773:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575258.006774:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575258.006776:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575258.006778:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575258.006783:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006787:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575258.006788:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006789:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575258.006790:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575258.006792:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006796:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575258.006798:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006802:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575258.006804:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006805:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575258.006805:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006806:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575258.006808:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575258.006809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575258.006811:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575258.006813:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575258.006815:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575258.006820:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393196160:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575258.006822:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006823:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575258.006826:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393196032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006830:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575258.006831:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575258.006833:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575258.006834:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575258.006835:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575258.006836:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575258.006837:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575258.006839:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:3.0:1788575258.006840:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575258.006841:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575258.006842:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:3.0:1788575258.006844:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575258.006845:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006846:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575258.006848:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393196160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575258.006853:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575258.006853:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575258.006854:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575258.006855:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575258.006856:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575258.006857:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575258.006858:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575258.006859:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:3.0:1788575258.006860:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575258.006861:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575258.006861:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:3.0:1788575258.006863:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575258.006863:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006864:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575258.006867:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575258.006868:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575258.006869:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575258.006870:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575258.389492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575258.389497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575258.389501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575258.389502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575258.389505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575258.389506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575258.965472:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306862521, at 4306862528
00000100:00000040:1.0:1788575258.965472:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306862521, at 4306862528
00000100:00000010:0.0:1788575258.965487:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:1.0:1788575258.965490:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575259.413529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575259.413535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575259.413539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575259.413540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575259.413542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575259.413543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575260.437551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575260.437558:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575260.437563:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575260.437564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575260.437567:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575260.437568:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575260.437649:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575260.437654:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575260.437657:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00000001:3.0:1788575261.461521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575261.461530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575261.461538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575261.461539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575261.461544:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575261.461546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575262.485509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575262.485530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575262.485536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575262.485537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575262.485541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575262.485542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575263.125723:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575263.125732:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575263.125737:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575263.125758:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.125816:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575263.125826:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575263.125840:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575263.125847:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575263.125854:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575263.125870:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393196288:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575263.125888:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575263.125898:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575263.125906:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575263.125919:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575263.125931:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575263.125938:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575263.125961:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575263.125967:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575263.125977:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575263.125989:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393196288, portal 25
00000100:00000001:3.0:1788575263.125993:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575263.126000:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:3.0:1788575263.126018:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.126041:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575263.126046:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575263.126051:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575263.126059:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393196288, offset 0
00000400:00000200:3.0:1788575263.126084:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575263.126104:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575263.126120:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575263.126139:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575263.126152:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575263.126170:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575263.126184:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575263.126196:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330789:330789:256:4294967295] 192.168.202.110@tcp LPNI seq info [330746:330746:8:4294967295]
00000400:00000200:3.0:1788575263.126215:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575263.126249:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575263.126273:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575263.126280:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575263.126298:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575263.126302:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788575263.126308:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575263.126311:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1788575263.126315:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575263.126317:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575263.126325:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575263.126337:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575263.126343:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575263.126346:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575263.126351:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575263.126354:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.126363:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575263.126366:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.126372:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575263.126373:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575263.126380:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575263.126384:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575263.126386:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306867641
00000100:00000010:3.0:1788575263.127555:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575263.127563:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575263.127574:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306867643
00000100:00000010:3.0:1788575263.127578:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575263.127580:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575263.127582:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575263.127585:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575263.127588:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575263.127595:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575263/real 1788575263]  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575263.127612:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575263.127613:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575263.127616:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575263.127621:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575263.127624:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575263.127627:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575263.127630:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575263.127637:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127641:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575263.127643:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.127645:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575263.127646:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575263.127649:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127654:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575263.127657:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127662:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575263.127664:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.127665:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575263.127666:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.127668:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575263.127670:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575263.127672:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575263.127674:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575263.127676:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575263.127678:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575263.127684:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393196288:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575263.127694:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.127697:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575263.127702:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393196288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127708:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575263.127709:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575263.127711:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575263.127715:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575263.127716:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575263.127717:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575263.127719:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575263.127721:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575263.127724:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575263.127726:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575263.127727:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:3.0:1788575263.127729:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575263.127730:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.127732:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.127738:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575263.127739:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575263.127741:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575263.127743:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575263.127744:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575263.127746:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127752:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575263.127753:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575263.127755:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575263.127757:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575263.127759:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575263.127761:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393196416:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575263.127764:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575263.127767:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575263.127768:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575263.127770:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575263.127773:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575263.127775:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575263.127779:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575263.127781:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575263.127785:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575263.127789:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393196416, portal 10
00000100:00000001:3.0:1788575263.127791:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575263.127792:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:3.0:1788575263.127797:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127802:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575263.127804:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575263.127806:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575263.127809:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393196416, offset 0
00000400:00000200:3.0:1788575263.127813:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575263.127818:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575263.127823:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575263.127827:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575263.127830:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575263.127833:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575263.127836:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575263.127839:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330790:330790:256:4294967295] 192.168.202.110@tcp LPNI seq info [330747:330747:8:4294967295]
00000400:00000200:3.0:1788575263.127843:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575263.127850:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575263.127853:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575263.127857:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575263.127862:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575263.127865:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788575263.127869:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575263.127871:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1788575263.127874:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575263.127876:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575263.127879:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575263.127882:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575263.127886:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575263.127888:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575263.127889:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575263.127892:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127897:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575263.127899:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127904:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575263.127905:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575263.127909:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575263.127912:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575263.127914:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575263.127916:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306866728
00000100:00000001:3.0:1788575263.127918:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306867641
00000100:00000001:3.0:1788575263.127920:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306866728
00000100:00000001:3.0:1788575263.127921:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306867643
00000100:00000001:3.0:1788575263.127922:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575263.127924:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575263.127925:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575263.127927:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575263/real 1788575263]  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575263.127932:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575263.127933:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575263.127934:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575263.127936:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575263.127937:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575263.127938:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575263.127940:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575263.127945:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127949:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575263.127950:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.127951:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575263.127952:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575263.127954:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127958:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575263.127961:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127965:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575263.127967:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.127968:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575263.127969:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.127970:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575263.127972:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575263.127973:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575263.127975:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575263.127977:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575263.127979:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575263.127984:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393196416:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575263.127987:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575263.127989:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.127994:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575263.127995:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575263.127996:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575263.127997:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575263.127999:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575263.128000:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393196544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575263.128002:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575263.128003:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575263.128004:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575263.128006:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575263.128008:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575263.128010:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575263.128012:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575263.128014:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575263.128016:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575263.128018:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393196544, portal 4
00000100:00000001:3.0:1788575263.128020:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575263.128021:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:3.0:1788575263.128025:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.128030:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575263.128031:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575263.128033:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575263.128034:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393196544, offset 0
00000400:00000200:3.0:1788575263.128037:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575263.128039:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575263.128041:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575263.128043:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575263.128045:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575263.128046:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575263.128048:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575263.128050:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330791:330791:256:4294967295] 192.168.202.110@tcp LPNI seq info [330748:330748:8:4294967295]
00000400:00000200:3.0:1788575263.128053:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575263.128057:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575263.128060:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575263.128061:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575263.128064:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575263.128066:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788575263.128069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575263.128070:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1788575263.128073:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575263.128074:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575263.128076:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575263.128078:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575263.128080:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575263.128081:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575263.128083:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575263.128085:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.128090:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575263.128092:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.128096:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575263.128097:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575263.128099:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575263.128101:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575263.128102:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575263.128104:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306866874
00000100:00000001:3.0:1788575263.128105:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306867641
00000100:00000001:3.0:1788575263.128107:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306866874
00000100:00000001:3.0:1788575263.128109:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306867643
00000100:00000001:3.0:1788575263.128110:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575263.128111:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575263.128113:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575263.128115:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575263/real 1788575263]  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575263.128119:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575263.128120:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575263.128121:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575263.128122:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575263.128123:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575263.128125:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575263.128127:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575263.128131:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.128136:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575263.128136:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.128138:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575263.128139:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575263.128141:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.128145:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575263.128147:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.128151:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575263.128153:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.128154:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575263.128154:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.128156:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575263.128157:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575263.128159:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575263.128161:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575263.128163:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575263.128165:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575263.128170:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393196544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575263.128172:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.128173:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575263.128175:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393196416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.128179:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575263.128180:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575263.128181:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575263.128182:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575263.128183:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575263.128184:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575263.128186:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575263.128187:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575263.128188:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575263.128189:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575263.128190:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:3.0:1788575263.128192:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575263.128193:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.128194:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575263.128196:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393196544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575263.128200:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575263.128200:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575263.128201:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575263.128203:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575263.128204:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575263.128205:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575263.128206:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575263.128207:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575263.128209:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575263.128209:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575263.128210:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:3.0:1788575263.128211:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575263.128212:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.128213:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575263.128215:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575263.128216:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575263.128217:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575263.128218:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575263.509527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575263.509537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575263.509545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575263.509551:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575263.509553:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575263.509558:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575263.509563:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575263.509566:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575263.509568:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575263.509571:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575263.509576:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575263.509579:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575263.509580:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575263.509581:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575263.509583:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575263.509584:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575263.509585:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575263.509587:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575263.509662:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575263.509668:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575263.509672:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1788575264.085488:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306867643, at 4306867648
00000100:00000040:0.0:1788575264.085495:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306867641, at 4306867648
00000100:00000010:1.0:1788575264.085507:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788575264.085509:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00010000:00000001:3.0:1788575264.533507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575264.533511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575264.533515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575264.533516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575264.533519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575264.533521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575265.557530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575265.557540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575265.557549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575265.557550:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575265.557555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575265.557556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575266.581523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575266.581531:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575266.581534:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575266.581537:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575266.581542:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575266.581544:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575266.581546:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575266.581547:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575266.581551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575266.581552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575266.581555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575266.581556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575266.581626:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575266.581631:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575266.581635:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00010000:00000001:3.0:1788575267.605578:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575267.605587:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575267.605595:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575267.605597:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575267.605601:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575267.605603:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575268.245546:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575268.245559:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575268.245562:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575268.245566:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575268.245571:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575268.245578:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12199
00000400:00000200:3.0:1788575268.245589:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575268.245595:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575268.245598:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575268.245600:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245605:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575268.245606:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245609:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575268.245613:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575268.245615:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575268.245617:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575268.245620:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575268.245622:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575268.245623:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575268.245625:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245627:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575268.245629:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245630:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575268.245633:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575268.245635:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245637:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575268.245640:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575268.245644:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575268.245646:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575268.245648:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575268.245650:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575268.245661:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000020:00000040:3.0:1788575268.245664:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575268.245668:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575268.245669:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575268.245672:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575268.245675:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575268.245679:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575268.245686:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8c00.
00000100:00000001:3.0:1788575268.245693:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575268.245703:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393196672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575268.245722:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575268.245727:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575268.245733:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393196672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575268.245763:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575268.245767:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575268.245771:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575268.245772:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575268.245775:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575268.245778:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575268.245782:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12199
00000400:00000200:3.0:1788575268.245788:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575268.245790:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575268.245792:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575268.245796:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245798:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575268.245802:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245806:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575268.245808:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575268.245810:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575268.245815:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575268.245818:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575268.245823:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575268.245826:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575268.245830:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245832:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575268.245838:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245840:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575268.245846:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575268.245858:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575268.245865:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575268.245875:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575268.245893:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575268.245897:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575268.245902:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575268.245908:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575268.245915:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575268.245918:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575268.245924:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575268.245928:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575268.245929:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575268.245932:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575268.245937:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575268.245938:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575268.245940:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575268.245946:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000100:00000001:3.0:1788575268.245951:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575268.245960:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393196800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575268.245976:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575268.245978:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575268.245985:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393196800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575268.246008:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575268.246013:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575268.246027:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575268.246031:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575268.246036:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575268.246045:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575268.246054:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12199
00000400:00000200:3.0:1788575268.246066:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575268.246075:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575268.246079:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575268.246085:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.246090:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575268.246101:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.246103:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575268.246110:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575268.246119:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575268.246123:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575268.246127:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575268.246131:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575268.246153:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575268.246159:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.246162:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575268.246169:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575268.246172:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575268.246176:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575268.246183:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575268.246193:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575268.246207:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575268.246215:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575268.246223:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575268.246231:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575268.246238:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575268.246253:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575268.246257:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575268.246262:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575268.246266:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575268.246275:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575268.246279:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575268.246292:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:2.1:1788575268.246298:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575268.246301:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
02000000:00000001:3.0:1788575268.246308:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575268.246311:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575268.246321:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:3.0:1788575268.246326:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575268.246352:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575268.246452:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575268.246460:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575268.246477:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575268.246507:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575268.246513:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575268.246520:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575268.246528:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12209)
00000800:00000200:2.1:1788575268.246818:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575268.246821:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788575268.247051:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575268.247052:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000400:00000010:2.0:1788575268.247064:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575268.247066:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575268.247068:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575268.247073:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575268.247075:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575268.247079:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575268.247081:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575268.247082:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575268.247085:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575268.247086:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575268.247088:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306871993
00000100:00000001:2.0:1788575268.247090:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306872762
00000100:00000001:2.0:1788575268.247091:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306871993
00000100:00000001:2.0:1788575268.247093:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306872762
00000100:00000001:2.0:1788575268.247094:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575268.247095:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575268.247096:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575268.247099:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575268/real 1788575268]  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575268.247103:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575268.247104:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575268.247105:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575268.247106:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575268.247107:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575268.247108:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575268.247110:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575268.247115:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575268.247123:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575268.247124:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575268.247126:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575268.247127:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575268.247129:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575268.247133:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575268.247135:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575268.247156:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575268.247158:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575268.247159:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575268.247160:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575268.247161:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575268.247162:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575268.247164:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575268.247166:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575268.247169:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575268.247171:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575268.247177:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393196928:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575268.247179:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575268.247180:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575268.247182:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393196800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575268.247186:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575268.247187:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575268.247189:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575268.247191:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575268.247192:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575268.247193:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575268.247194:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575268.247196:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:2.0:1788575268.247197:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575268.247198:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575268.247199:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575268.247202:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575268.247203:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575268.247204:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575268.247206:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393196928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575268.247210:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575268.247211:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575268.247212:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575268.247213:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575268.247214:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575268.247214:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575268.247215:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575268.247216:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575268.247218:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575268.247218:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575268.247219:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575268.247220:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575268.247221:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575268.247222:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575268.247225:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575268.247226:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575268.247227:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575268.247228:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575268.629525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575268.629533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575268.629538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575268.629539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575268.629542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575268.629543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575268.629614:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575268.629618:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575268.629623:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575268.757464:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575268.757473:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575268.757477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575268.757483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575268.757488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575268.757493:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575268.757500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575268.757503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575268.757507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575269.205524:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306872762, at 4306872768
00000100:00000040:0.0:1788575269.205537:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306872762, at 4306872768
00000100:00000010:1.0:1788575269.205538:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000100:00000010:0.0:1788575269.205554:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00010000:00000001:3.0:1788575269.653516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575269.653523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575269.653528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575269.653529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575269.653533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575269.653534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575269.653600:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575269.653605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575269.653608:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575269.781563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575269.781571:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575269.781573:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575269.781579:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575269.781582:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575269.781587:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575269.781589:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575269.781592:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575269.781594:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575270.677481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575270.677487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575270.677491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575270.677492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575270.677494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575270.677495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575270.677585:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575270.677591:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575270.677597:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575270.805549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575270.805558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575270.805561:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575270.805566:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575270.805570:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575270.805574:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575270.805577:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575270.805579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575270.805582:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575271.701570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575271.701579:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575271.701587:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575271.701588:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575271.701593:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575271.701594:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575271.701697:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575271.701705:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575271.701709:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575271.829545:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575271.829555:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575271.829558:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575271.829565:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575271.829569:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575271.829574:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575271.829578:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575271.829581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575271.829585:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575272.725525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575272.725532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575272.725536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575272.725538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575272.725541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575272.725542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575272.725601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575272.725604:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575272.725607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575272.853519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575272.853526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575272.853528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575272.853533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575272.853537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575272.853541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575272.853543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575272.853546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575272.853549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575273.365503:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575273.365513:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575273.365516:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575273.365519:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575273.365523:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575273.365530:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12204
00000400:00000200:3.0:1788575273.365538:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575273.365544:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575273.365547:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575273.365549:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365551:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575273.365553:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365555:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575273.365558:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575273.365559:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575273.365562:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575273.365564:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575273.365565:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575273.365567:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575273.365568:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365570:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575273.365572:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365573:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575273.365575:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575273.365578:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365579:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575273.365582:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575273.365584:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575273.365586:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575273.365588:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575273.365590:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575273.365599:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000020:00000040:3.0:1788575273.365602:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575273.365605:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575273.365606:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575273.365608:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575273.365611:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575273.365614:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575273.365621:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788575273.365628:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575273.365635:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575273.365643:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575273.365647:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575273.365650:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575273.365669:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575273.365671:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575273.365674:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575273.365676:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575273.365677:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575273.365679:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575273.365681:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12204
00000400:00000200:3.0:1788575273.365683:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575273.365685:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575273.365687:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575273.365697:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365699:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575273.365704:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365706:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575273.365708:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575273.365708:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575273.365709:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575273.365712:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575273.365713:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575273.365714:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575273.365717:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575273.365723:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575273.365725:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575273.365725:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575273.365727:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575273.365732:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575273.365738:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575273.365740:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575273.365742:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365744:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575273.365746:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575273.365752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575273.365753:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575273.365758:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575273.365763:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575273.365769:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788575273.365775:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:3.0:1788575273.365776:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575273.365778:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575273.365780:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575273.365784:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575273.365784:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575273.365794:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575273.365795:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575273.365801:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575273.365803:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575273.365804:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575273.365809:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575273.365810:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575273.365815:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575273.365822:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393197056:192.168.202.110@tcp:250:kworker.0
02000000:00000010:3.0:1788575273.365823:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:2.0:1788575273.365834:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575273.365835:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575273.365839:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575273.365843:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575273.365843:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575273.365849:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575273.365856:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575273.365859:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575273.365860:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575273.365873:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575273.365878:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575273.365886:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575273.365894:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575273.365901:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000001:3.0:1788575273.365903:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575273.365906:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1788575273.365907:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393197056, portal 25
00000100:00080000:3.0:1788575273.365917:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575273.365918:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575273.365922:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000001:3.0:1788575273.365923:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575273.365931:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575273.365938:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:2.0:1788575273.365939:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575273.365951:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12204
00000100:00000001:2.0:1788575273.365957:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:3.0:1788575273.365962:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575273.365965:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575273.365971:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:2.0:1788575273.365972:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00080000:3.0:1788575273.365985:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575273.365990:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365991:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575273.365993:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.365994:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575273.365996:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575273.365997:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575273.365998:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575273.365999:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575273.366000:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575273.366006:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:2.0:1788575273.366009:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393197056, offset 0
00000100:00000040:3.0:1788575273.366010:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.366016:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575273.366020:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575273.366021:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575273.366023:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575273.366024:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575273.366026:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575273.366027:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575273.366030:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575273.366034:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575273.366037:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575273.366037:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575273.366039:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575273.366043:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575273.366044:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575273.366047:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000020:00000040:3.0:1788575273.366049:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575273.366050:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575273.366054:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575273.366055:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575273.366060:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575273.366062:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575273.366064:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575273.366066:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575273.366069:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575273.366074:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575273.366076:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575273.366090:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788575273.366099:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575273.366100:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575273.366111:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575273.366114:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575273.366119:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:3.0:1788575273.366120:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575273.366136:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330795:330795:256:4294967295] 192.168.202.110@tcp LPNI seq info [330752:330752:8:4294967295]
00000100:00000001:3.0:1788575273.366138:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575273.366144:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575273.366153:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575273.366154:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575273.366161:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12214)
00000400:00000200:2.0:1788575273.366169:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575273.366178:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575273.366184:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575273.366191:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575273.366194:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575273.366201:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575273.366205:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575273.366208:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575273.366210:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575273.366215:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575273.366218:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575273.366222:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575273.366224:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575273.366227:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575273.366230:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366237:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575273.366240:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366248:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575273.366249:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575273.366253:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575273.366256:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575273.366258:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306877881
00000100:00000010:2.0:1788575273.366263:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cf00.
00000100:00000001:2.0:1788575273.366269:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575273.366287:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306877881
00000100:00000010:2.0:1788575273.366291:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c600.
00000100:00000001:2.0:1788575273.366294:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575273.366297:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575273.366298:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575273.366300:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575273.366304:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575273/real 1788575273]  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575273.366311:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575273.366312:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575273.366314:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575273.366316:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575273.366317:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575273.366319:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575273.366321:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575273.366327:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366331:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575273.366332:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366333:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575273.366334:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575273.366337:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366342:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575273.366344:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366348:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575273.366350:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366351:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575273.366352:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366353:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575273.366355:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575273.366357:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575273.366360:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575273.366361:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575273.366363:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575273.366368:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393197056:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575273.366371:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366372:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575273.366374:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393197056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366378:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575273.366380:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575273.366382:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575273.366384:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575273.366387:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575273.366388:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575273.366426:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575273.366429:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575273.366433:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575273.366436:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575273.366438:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575273.366441:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575273.366442:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366444:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366452:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575273.366454:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575273.366456:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575273.366458:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575273.366459:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575273.366464:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366472:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575273.366473:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575273.366476:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575273.366478:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575273.366480:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575273.366483:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197184:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575273.366486:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575273.366489:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575273.366491:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575273.366492:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575273.366495:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575273.366497:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575273.366501:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575273.366504:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575273.366508:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575273.366511:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393197184, portal 10
00000100:00000001:2.0:1788575273.366513:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575273.366514:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788575273.366521:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366527:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575273.366529:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575273.366532:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575273.366535:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393197184, offset 0
00000400:00000200:2.0:1788575273.366538:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575273.366542:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575273.366545:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575273.366548:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575273.366558:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575273.366560:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575273.366563:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575273.366565:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330796:330796:256:4294967295] 192.168.202.110@tcp LPNI seq info [330753:330753:8:4294967295]
00000400:00000200:2.0:1788575273.366570:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575273.366575:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575273.366580:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575273.366583:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575273.366587:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575273.366590:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575273.366594:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575273.366597:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575273.366601:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575273.366602:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575273.366604:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575273.366606:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575273.366608:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575273.366609:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575273.366611:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575273.366613:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366618:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575273.366620:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366624:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575273.366625:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575273.366627:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575273.366630:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575273.366632:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575273.366633:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306876967
00000100:00000001:2.0:1788575273.366635:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306877881
00000100:00000001:2.0:1788575273.366637:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306876967
00000100:00000001:2.0:1788575273.366638:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306877881
00000100:00000001:2.0:1788575273.366639:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575273.366640:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575273.366641:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575273.366644:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575273/real 1788575273]  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575273.366649:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575273.366650:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575273.366651:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575273.366652:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575273.366653:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575273.366655:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575273.366656:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575273.366661:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366664:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575273.366665:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366666:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575273.366667:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575273.366669:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366672:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575273.366675:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366679:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575273.366680:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366681:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575273.366682:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366683:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575273.366685:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575273.366687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575273.366690:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575273.366692:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575273.366696:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575273.366704:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197184:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575273.366707:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575273.366711:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366718:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575273.366719:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575273.366721:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575273.366723:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575273.366725:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575273.366728:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197312:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575273.366731:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575273.366733:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575273.366735:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575273.366736:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575273.366741:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575273.366743:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575273.366747:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575273.366750:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575273.366753:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575273.366756:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393197312, portal 4
00000100:00000001:2.0:1788575273.366758:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575273.366759:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575273.366765:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366771:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575273.366773:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575273.366776:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575273.366778:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393197312, offset 0
00000400:00000200:2.0:1788575273.366782:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575273.366786:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575273.366789:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575273.366792:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575273.366794:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575273.366797:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575273.366799:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575273.366802:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330797:330797:256:4294967295] 192.168.202.110@tcp LPNI seq info [330754:330754:8:4294967295]
00000400:00000200:2.0:1788575273.366806:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575273.366812:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575273.366816:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575273.366819:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575273.366822:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575273.366825:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575273.366829:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575273.366831:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575273.366835:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575273.366836:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575273.366839:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575273.366842:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575273.366845:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575273.366847:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575273.366849:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575273.366852:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366859:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575273.366862:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366868:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575273.366869:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575273.366872:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575273.366874:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575273.366877:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575273.366878:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306877113
00000100:00000001:2.0:1788575273.366879:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306877881
00000100:00000001:2.0:1788575273.366881:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306877113
00000100:00000001:2.0:1788575273.366882:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306877881
00000100:00000001:2.0:1788575273.366883:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575273.366884:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575273.366885:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575273.366888:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575273/real 1788575273]  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575273.366892:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575273.366893:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575273.366894:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575273.366895:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575273.366896:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575273.366897:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575273.366899:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575273.366904:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366911:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575273.366912:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366913:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575273.366914:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575273.366918:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366924:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575273.366928:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366934:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575273.366936:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366938:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575273.366939:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366941:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575273.366942:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575273.366945:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575273.366948:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575273.366950:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575273.366954:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575273.366962:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197312:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575273.366966:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.366968:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575273.366971:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393197184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.366987:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575273.366989:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575273.366991:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575273.366994:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575273.366995:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575273.366997:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575273.366999:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575273.367000:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575273.367003:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575273.367004:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575273.367005:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575273.367007:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575273.367009:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.367010:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575273.367014:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393197312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575273.367020:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575273.367021:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575273.367022:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575273.367024:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575273.367025:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575273.367026:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575273.367028:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575273.367029:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575273.367032:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575273.367033:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575273.367034:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575273.367036:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575273.367037:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.367038:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.367041:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575273.367043:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575273.367044:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575273.367046:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575273.749555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575273.749562:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575273.749566:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575273.749569:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575273.749571:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575273.749574:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575273.749577:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575273.749579:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575273.749580:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575273.749582:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575273.749584:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575273.749585:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575273.749586:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575273.749587:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575273.749588:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575273.749589:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575273.749590:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575273.749591:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575273.749673:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575273.749678:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575273.749681:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575273.877496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575273.877501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575273.877503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575273.877506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575273.877509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575273.877512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.877514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575273.877516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575273.877518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575274.325538:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306877881, at 4306877888
00000100:00000040:1.0:1788575274.325538:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306877881, at 4306877888
00000100:00000010:1.0:1788575274.325559:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000100:00000010:0.0:1788575274.325561:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00010000:00000001:3.0:1788575274.773533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575274.773542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575274.773549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575274.773551:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575274.773554:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575274.773555:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575274.773616:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575274.773620:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575274.773623:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575274.901537:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575274.901544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575274.901546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575274.901551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575274.901554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575274.901558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575274.901560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575274.901562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575274.901565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575275.797493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575275.797502:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575275.797505:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575275.797509:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575275.797515:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575275.797517:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575275.797520:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575275.797522:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575275.797527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575275.797529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575275.797533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575275.797534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575275.797601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575275.797606:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575275.797610:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575275.925507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575275.925515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575275.925516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575275.925521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575275.925524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575275.925528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575275.925530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575275.925532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575275.925535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575276.821502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575276.821506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575276.821509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575276.821510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575276.821513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575276.821514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575276.821577:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575276.821582:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575276.821586:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575276.949535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575276.949543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575276.949545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575276.949550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575276.949553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575276.949557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575276.949559:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575276.949562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575276.949565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575277.845510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575277.845516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575277.845521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575277.845523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575277.845527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575277.845528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575277.845598:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575277.845603:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575277.845608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575277.973532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575277.973541:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575277.973544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575277.973550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575277.973555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575277.973560:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575277.973562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575277.973565:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575277.973569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575278.485500:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575278.485508:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575278.485510:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575278.485513:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575278.485517:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575278.485522:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12209
00000400:00000200:3.0:1788575278.485531:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575278.485537:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575278.485540:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575278.485543:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575278.485546:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575278.485548:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575278.485550:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575278.485554:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575278.485556:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575278.485559:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575278.485562:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575278.485564:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575278.485567:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575278.485570:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575278.485573:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575278.485575:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575278.485577:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575278.485580:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575278.485582:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575278.485584:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575278.485586:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575278.485589:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575278.485591:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575278.485592:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575278.485594:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575278.485604:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2300.
00000020:00000040:3.0:1788575278.485608:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575278.485613:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575278.485614:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575278.485617:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575278.485620:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575278.485623:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575278.485631:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575278.485638:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575278.485647:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575278.485657:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575278.485661:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575278.485665:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc4bce75] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575278.485687:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575278.485691:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575278.485694:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575278.485695:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575278.485698:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575278.485701:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575278.485704:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12209
00000400:00000200:3.0:1788575278.485708:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575278.485711:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575278.485713:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575278.485716:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575278.485719:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575278.485723:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575278.485726:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575278.485728:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575278.485730:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575278.485732:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575278.485735:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575278.485737:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575278.485739:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575278.485744:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575278.485745:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575278.485752:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575278.485753:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575278.485757:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575278.485758:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575278.485764:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575278.485768:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:2.0:1788575278.485772:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575278.485776:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575278.485787:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575278.485795:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575278.485804:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1788575278.485805:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575278.485809:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575278.485811:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575278.485826:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575278.485829:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:3.0:1788575278.485831:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575278.485833:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1788575278.485836:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575278.485838:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575278.485840:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575278.485847:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393197440:192.168.202.110@tcp:250:kworker.0
00000100:00000010:3.0:1788575278.485848:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0a80.
00000100:00000001:2.0:1788575278.485860:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1788575278.485860:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575278.485867:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575278.485868:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575278.485869:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575278.485871:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575278.485871:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575278.485879:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575278.485879:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575278.485888:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575278.485890:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575278.485897:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575278.485901:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575278.485904:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575278.485907:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575278.485915:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575278.485925:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000100:00000001:3.0:1788575278.485935:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1788575278.485938:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000001:3.0:1788575278.485938:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575278.485948:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393197440, portal 25
00000100:00000040:3.0:1788575278.485949:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b298c26f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.485953:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575278.485956:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000040:2.0:1788575278.485967:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.485984:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575278.485985:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575278.485989:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575278.485994:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00080000:3.0:1788575278.485996:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1788575278.485998:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393197440, offset 0
00000100:00080000:3.0:1788575278.486007:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:2.0:1788575278.486014:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575278.486014:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575278.486024:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575278.486027:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575278.486032:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12209
00000400:00000200:2.0:1788575278.486038:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575278.486048:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575278.486052:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575278.486056:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575278.486059:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575278.486062:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575278.486065:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330798:330798:256:4294967295] 192.168.202.110@tcp LPNI seq info [330755:330755:8:4294967295]
00000400:00000200:2.0:1788575278.486068:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575278.486073:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575278.486084:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575278.486090:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575278.486097:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1788575278.486101:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575278.486106:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1788575278.486112:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575278.486114:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575278.486122:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000100:00000001:3.0:1788575278.486129:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.0:1788575278.486135:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788575278.486137:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575278.486140:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:2.0:1788575278.486141:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:3.0:1788575278.486142:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575278.486146:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:2.1:1788575278.486152:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788575278.486154:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.1:1788575278.486158:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:3.0:1788575278.486161:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000100:2.0:1788575278.486165:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575278.486170:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000100:00000040:3.0:1788575278.486175:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575278.486178:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1788575278.486180:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575278.486185:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1788575278.486191:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575278.486194:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000100:00000001:3.0:1788575278.486197:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:2.0:1788575278.486198:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575278.486205:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000020:00000040:3.0:1788575278.486206:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000200:2.0:1788575278.486211:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575278.486215:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575278.486226:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575278.486234:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575278.486239:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575278.486249:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486253:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1788575278.486253:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575278.486258:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000040:2.0:1788575278.486262:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:3.0:1788575278.486262:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575278.486271:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575278.486282:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575278.486283:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486293:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000040:3.0:1788575278.486293:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575278.486299:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575278.486301:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486304:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575278.486305:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575278.486312:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575278.486315:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575278.486316:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575278.486323:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306883001
02000000:00000010:3.0:1788575278.486324:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575278.486328:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575278.486330:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c4e0.
00000100:00100000:3.0:1788575278.486335:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575278.486339:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575278.486356:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575278.486359:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486362:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306883001
00000100:00000040:3.0:1788575278.486365:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1788575278.486367:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ccc0.
00000100:00000001:3.0:1788575278.486379:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486380:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1788575278.486431:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575278.486436:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1788575278.486437:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575278.486445:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486449:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1788575278.486449:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12219)
00000100:00100000:2.0:1788575278.486456:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575278/real 1788575278]  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575278.486466:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575278.486468:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575278.486470:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575278.486472:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575278.486474:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575278.486477:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575278.486481:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575278.486494:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486500:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575278.486501:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.486503:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575278.486505:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575278.486508:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486512:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575278.486515:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486519:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575278.486521:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.486522:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575278.486523:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.486525:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575278.486528:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575278.486531:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575278.486535:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575278.486538:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575278.486542:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575278.486550:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393197440:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575278.486556:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.486558:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575278.486561:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393197440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486568:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575278.486571:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575278.486574:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575278.486578:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575278.486580:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575278.486582:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575278.486584:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575278.486586:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575278.486590:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575278.486591:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575278.486593:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:2.0:1788575278.486595:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575278.486596:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.486598:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.486605:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575278.486607:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486608:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575278.486609:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575278.486610:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575278.486613:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486618:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575278.486620:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575278.486621:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575278.486623:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575278.486624:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575278.486626:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575278.486628:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575278.486629:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575278.486631:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575278.486632:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575278.486634:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575278.486635:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575278.486638:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575278.486640:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575278.486642:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575278.486645:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393197568, portal 10
00000100:00000001:2.0:1788575278.486646:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575278.486647:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:2.0:1788575278.486652:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486657:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575278.486658:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575278.486659:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575278.486661:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393197568, offset 0
00000400:00000200:2.0:1788575278.486663:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575278.486666:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575278.486668:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575278.486670:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575278.486672:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575278.486674:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575278.486675:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575278.486677:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330799:330799:256:4294967295] 192.168.202.110@tcp LPNI seq info [330756:330756:8:4294967295]
00000400:00000200:2.0:1788575278.486680:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575278.486685:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575278.486688:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575278.486690:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575278.486692:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575278.486694:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575278.486697:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575278.486698:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575278.486700:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575278.486702:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575278.486704:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575278.486706:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575278.486708:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575278.486709:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575278.486711:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575278.486713:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486718:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575278.486720:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486728:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486729:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575278.486731:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575278.486734:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575278.486736:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575278.486738:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306882087
00000100:00000001:2.0:1788575278.486740:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306883001
00000100:00000001:2.0:1788575278.486741:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306882087
00000100:00000001:2.0:1788575278.486743:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306883001
00000100:00000001:2.0:1788575278.486744:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575278.486746:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486747:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575278.486749:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575278/real 1788575278]  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575278.486753:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575278.486753:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575278.486754:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575278.486755:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575278.486756:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575278.486757:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575278.486759:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575278.486764:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486768:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575278.486769:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.486769:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575278.486770:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575278.486772:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486777:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575278.486779:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486783:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575278.486784:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.486785:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575278.486786:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.486787:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575278.486789:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575278.486790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575278.486792:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575278.486793:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575278.486795:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575278.486799:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575278.486801:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575278.486803:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486808:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575278.486809:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575278.486810:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575278.486811:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575278.486812:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575278.486829:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575278.486831:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575278.486832:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575278.486834:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575278.486835:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575278.486838:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575278.486839:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575278.486842:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575278.486843:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575278.486846:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575278.486848:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393197696, portal 4
00000100:00000001:2.0:1788575278.486849:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575278.486850:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575278.486856:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486862:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575278.486864:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575278.486866:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575278.486868:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393197696, offset 0
00000400:00000200:2.0:1788575278.486871:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575278.486875:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575278.486878:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575278.486881:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575278.486884:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575278.486886:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575278.486888:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575278.486892:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330800:330800:256:4294967295] 192.168.202.110@tcp LPNI seq info [330757:330757:8:4294967295]
00000400:00000200:2.0:1788575278.486896:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575278.486901:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575278.486904:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575278.486906:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575278.486909:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575278.486910:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575278.486913:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575278.486914:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575278.486917:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575278.486918:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575278.486920:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575278.486922:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575278.486924:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575278.486926:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575278.486927:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575278.486929:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486935:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575278.486937:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486941:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486942:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575278.486943:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575278.486946:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575278.486948:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575278.486950:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306882233
00000100:00000001:2.0:1788575278.486953:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306883001
00000100:00000001:2.0:1788575278.486954:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306882233
00000100:00000001:2.0:1788575278.486957:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306883001
00000100:00000001:2.0:1788575278.486958:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575278.486960:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.486962:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575278.486966:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575278/real 1788575278]  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575278.486974:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575278.486975:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575278.486977:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575278.486980:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575278.486981:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575278.486984:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575278.486987:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575278.486994:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.486998:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575278.486999:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.487001:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575278.487002:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575278.487004:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.487009:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575278.487011:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.487014:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575278.487016:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.487017:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575278.487018:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.487019:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575278.487020:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575278.487022:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575278.487024:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575278.487025:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575278.487027:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575278.487031:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575278.487034:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.487035:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575278.487037:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393197568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.487041:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575278.487042:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575278.487044:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575278.487045:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575278.487046:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575278.487047:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575278.487049:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575278.487050:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575278.487052:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575278.487052:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575278.487053:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:2.0:1788575278.487055:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575278.487056:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.487057:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575278.487059:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393197696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575278.487062:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575278.487063:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575278.487064:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575278.487065:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575278.487066:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575278.487066:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575278.487067:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575278.487068:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575278.487069:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575278.487070:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575278.487071:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575278.487072:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575278.487073:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.487075:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.487077:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575278.487078:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575278.487079:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575278.487081:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575278.869499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575278.869506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575278.869511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575278.869512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575278.869515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575278.869516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575278.869569:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575278.869572:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575278.869574:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575278.997505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575278.997517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575278.997519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575278.997526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575278.997531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575278.997535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.997538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575278.997541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575278.997544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575279.445496:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306883001, at 4306883008
00000100:00000040:1.0:1788575279.445496:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306883001, at 4306883008
00000100:00000010:0.0:1788575279.445517:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000100:00000010:1.0:1788575279.445523:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00010000:00000001:3.0:1788575279.893555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575279.893564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575279.893571:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575279.893572:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575279.893576:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575279.893578:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575279.893632:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575279.893636:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575279.893640:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575280.021489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575280.021498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575280.021501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575280.021512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575280.021516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575280.021520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575280.021523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575280.021525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575280.021528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575280.917530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575280.917537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575280.917541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575280.917542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575280.917545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575280.917546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575280.917611:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575280.917615:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575280.917619:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575281.045526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575281.045533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575281.045535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575281.045540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575281.045543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575281.045547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575281.045549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575281.045551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575281.045554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575281.941551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575281.941557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575281.941562:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575281.941563:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575281.941566:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575281.941567:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575281.941627:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575281.941632:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575281.941636:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575282.069471:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575282.069479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575282.069481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575282.069485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575282.069489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575282.069493:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575282.069495:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575282.069498:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575282.069500:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575282.965471:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575282.965478:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575282.965482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575282.965483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575282.965486:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575282.965487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575282.965533:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575282.965536:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575282.965539:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575283.093521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575283.093529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575283.093531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575283.093536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575283.093539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575283.093543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.093546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575283.093548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.093550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575283.605535:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575283.605550:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575283.605553:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575283.605558:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575283.605563:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575283.605572:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12214
00000400:00000200:0.0:1788575283.605584:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575283.605593:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575283.605597:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575283.605601:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.605605:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575283.605607:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.605611:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575283.605616:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575283.605618:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575283.605622:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575283.605625:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575283.605627:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575283.605630:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575283.605633:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.605635:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575283.605637:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.605639:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575283.605642:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575283.605646:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575283.605649:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575283.605664:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575283.605668:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575283.605671:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575283.605674:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575283.605677:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575283.605696:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575283.605704:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575283.605708:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575283.605710:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575283.605714:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575283.605716:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575283.605721:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575283.605730:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788575283.605738:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575283.605750:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575283.605761:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575283.605766:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575283.605772:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575283.605797:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575283.605801:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575283.605806:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575283.605808:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575283.605809:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575283.605812:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575283.605815:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12214
00000400:00000200:0.0:1788575283.605820:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575283.605824:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575283.605826:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575283.605830:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.605833:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575283.605835:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.605838:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575283.605839:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788575283.605841:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575283.605845:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575283.605846:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788575283.605847:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575283.605851:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1788575283.605852:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575283.605856:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575283.605859:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575283.605862:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575283.605864:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.605868:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575283.605872:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.605877:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575283.605880:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1788575283.605881:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575283.605884:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575283.605887:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575283.605890:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575283.605891:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575283.605895:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575283.605895:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575283.605900:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575283.605905:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1788575283.605906:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393197824:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788575283.605911:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575283.605914:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1788575283.605915:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575283.605920:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:0.0:1788575283.605922:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575283.605925:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788575283.605926:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575283.605931:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575283.605932:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:0.0:1788575283.605936:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
02000000:00000010:2.0:1788575283.605940:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
00000020:00000040:0.0:1788575283.605941:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575283.605944:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575283.605946:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575283.605949:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575283.605953:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575283.605953:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575283.605956:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575283.605960:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575283.605960:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
02000000:00000010:0.0:1788575283.605970:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000400:00000010:2.0:1788575283.605970:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000001:0.0:1788575283.605974:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575283.605981:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393197824, portal 25
00000100:00100000:0.0:1788575283.605982:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575283.605986:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575283.605990:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000001:0.0:1788575283.605997:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575283.606000:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575283.606002:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575283.606007:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606020:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788575283.606025:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575283.606028:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788575283.606029:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:2.0:1788575283.606033:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00080000:0.0:1788575283.606034:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575283.606038:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575283.606040:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:2.0:1788575283.606040:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393197824, offset 0
00000100:00000001:0.0:1788575283.606046:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575283.606050:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12214
00000400:00000200:2.0:1788575283.606055:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575283.606057:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575283.606062:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575283.606064:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575283.606068:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575283.606071:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.606074:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575283.606076:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575283.606079:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788575283.606080:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575283.606084:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575283.606086:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575283.606087:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575283.606089:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575283.606091:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575283.606092:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575283.606097:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.606098:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575283.606100:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.606101:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575283.606103:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:0.0:1788575283.606105:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575283.606109:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575283.606112:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1788575283.606113:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575283.606116:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575283.606119:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575283.606120:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1788575283.606123:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575283.606126:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788575283.606128:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575283.606132:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575283.606136:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330801:330801:256:4294967295] 192.168.202.110@tcp LPNI seq info [330758:330758:8:4294967295]
00000008:00000001:0.0:1788575283.606137:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575283.606141:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575283.606148:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1788575283.606149:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788575283.606154:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575283.606156:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575283.606158:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575283.606161:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575283.606164:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575283.606167:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1788575283.606168:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575283.606176:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000800:00000200:2.0:1788575283.606179:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575283.606181:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575283.606188:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000100:00100000:0.0:1788575283.606189:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1788575283.606201:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575283.606203:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575283.606207:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575283.606208:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:0.0:1788575283.606215:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788575283.606220:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575283.606228:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1788575283.606233:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1788575283.606238:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788575283.606239:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:2.0:1788575283.606243:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000100:00000040:0.0:1788575283.606245:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575283.606252:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1788575283.606254:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12224)
00000400:00000200:2.0:1788575283.606261:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575283.606267:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575283.606270:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575283.606276:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575283.606281:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606291:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575283.606294:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606299:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575283.606300:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575283.606306:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575283.606309:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575283.606311:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306888121
00000100:00000010:2.0:1788575283.606316:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cae0.
00000100:00000001:2.0:1788575283.606323:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575283.606333:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306888121
00000100:00000010:2.0:1788575283.606335:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c660.
00000100:00000001:2.0:1788575283.606337:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575283.606340:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575283.606341:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575283.606343:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575283.606346:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575283/real 1788575283]  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575283.606354:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575283.606356:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575283.606358:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575283.606361:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575283.606363:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575283.606365:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575283.606369:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575283.606377:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606384:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575283.606387:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.606389:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575283.606420:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575283.606426:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606433:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575283.606438:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606445:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575283.606448:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.606450:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575283.606452:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.606454:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575283.606456:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575283.606459:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575283.606463:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575283.606466:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575283.606469:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575283.606477:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393197824:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575283.606482:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.606484:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575283.606487:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393197824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606493:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575283.606496:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575283.606499:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575283.606502:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575283.606504:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575283.606506:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575283.606509:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575283.606511:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5c00.
02000000:00000001:2.0:1788575283.606515:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575283.606517:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575283.606519:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:2.0:1788575283.606522:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575283.606523:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.606526:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.606534:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575283.606536:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575283.606538:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575283.606540:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575283.606541:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575283.606545:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606551:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575283.606553:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575283.606554:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575283.606556:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575283.606558:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575283.606561:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197952:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575283.606564:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575283.606565:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575283.606567:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575283.606568:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575283.606571:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575283.606573:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575283.606577:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575283.606579:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575283.606583:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575283.606585:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393197952, portal 10
00000100:00000001:2.0:1788575283.606587:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575283.606588:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
00000100:00000040:2.0:1788575283.606595:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606601:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575283.606603:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575283.606606:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575283.606608:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393197952, offset 0
00000400:00000200:2.0:1788575283.606612:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575283.606615:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575283.606619:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575283.606622:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575283.606624:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575283.606626:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575283.606628:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575283.606631:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330802:330802:256:4294967295] 192.168.202.110@tcp LPNI seq info [330759:330759:8:4294967295]
00000400:00000200:2.0:1788575283.606635:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575283.606641:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575283.606645:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575283.606648:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575283.606669:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575283.606672:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575283.606676:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575283.606678:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575283.606682:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575283.606684:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575283.606687:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575283.606690:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575283.606694:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575283.606695:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575283.606698:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575283.606701:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606709:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575283.606712:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606724:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575283.606726:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575283.606729:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575283.606733:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575283.606736:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575283.606738:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306887207
00000100:00000001:2.0:1788575283.606740:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306888121
00000100:00000001:2.0:1788575283.606743:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306887207
00000100:00000001:2.0:1788575283.606745:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306888121
00000100:00000001:2.0:1788575283.606746:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575283.606748:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575283.606750:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575283.606754:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575283/real 1788575283]  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575283.606762:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575283.606763:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575283.606765:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575283.606767:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575283.606769:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575283.606771:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575283.606774:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575283.606783:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606787:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575283.606788:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.606789:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575283.606790:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575283.606793:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606801:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575283.606805:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606811:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575283.606814:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.606815:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575283.606817:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.606818:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575283.606820:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575283.606823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575283.606826:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575283.606828:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575283.606832:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575283.606840:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393197952:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575283.606844:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575283.606847:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606853:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575283.606855:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575283.606856:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575283.606858:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575283.606860:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575283.606863:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198080:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575283.606867:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575283.606868:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575283.606871:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575283.606872:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575283.606876:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575283.606877:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575283.606881:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575283.606884:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575283.606887:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575283.606890:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393198080, portal 4
00000100:00000001:2.0:1788575283.606891:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575283.606893:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:2.0:1788575283.606899:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606905:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575283.606907:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575283.606909:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575283.606911:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393198080, offset 0
00000400:00000200:2.0:1788575283.606914:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575283.606917:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575283.606921:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575283.606923:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575283.606926:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575283.606928:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575283.606930:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575283.606933:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330803:330803:256:4294967295] 192.168.202.110@tcp LPNI seq info [330760:330760:8:4294967295]
00000400:00000200:2.0:1788575283.606937:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575283.606943:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575283.606947:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575283.606951:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575283.606954:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575283.606957:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575283.606961:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575283.606964:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575283.606967:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575283.606970:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575283.606973:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575283.606976:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575283.606979:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575283.606981:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575283.606983:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575283.606987:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.606994:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575283.606997:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.607003:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575283.607005:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575283.607008:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575283.607011:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575283.607014:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575283.607016:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306887353
00000100:00000001:2.0:1788575283.607018:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306888121
00000100:00000001:2.0:1788575283.607020:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306887353
00000100:00000001:2.0:1788575283.607023:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306888121
00000100:00000001:2.0:1788575283.607025:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575283.607027:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575283.607029:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575283.607033:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575283/real 1788575283]  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575283.607040:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575283.607041:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575283.607043:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575283.607046:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575283.607048:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575283.607050:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575283.607054:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575283.607062:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.607069:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575283.607070:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.607072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575283.607074:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575283.607077:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.607093:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575283.607100:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.607108:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575283.607110:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.607112:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575283.607114:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.607115:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575283.607117:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575283.607120:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575283.607123:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575283.607125:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575283.607129:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575283.607136:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198080:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575283.607139:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.607141:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575283.607144:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393197952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.607151:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575283.607153:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575283.607155:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575283.607157:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575283.607159:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575283.607160:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575283.607162:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575283.607164:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:2.0:1788575283.607166:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575283.607168:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575283.607169:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:2.0:1788575283.607172:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575283.607174:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.607176:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575283.607179:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393198080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575283.607185:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575283.607186:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575283.607187:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575283.607189:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575283.607191:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575283.607192:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575283.607193:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575283.607195:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:2.0:1788575283.607197:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575283.607198:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575283.607199:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:2.0:1788575283.607201:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575283.607203:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.607205:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575283.607208:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575283.607209:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575283.607211:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575283.607213:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575283.989514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575283.989524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575283.989530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575283.989532:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575283.989534:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575283.989538:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575283.989540:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575283.989542:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575283.989544:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575283.989546:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575283.989550:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575283.989551:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575283.989552:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575283.989554:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575283.989555:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575283.989556:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575283.989558:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575283.989559:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575283.989602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575283.989605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575283.989607:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575284.117490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575284.117498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575284.117501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575284.117506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575284.117510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575284.117513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575284.117516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575284.117518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575284.117520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575284.565499:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306888121, at 4306888128
00000100:00000040:0.0:1788575284.565511:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306888121, at 4306888128
00000100:00000010:1.0:1788575284.565515:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000100:00000010:0.0:1788575284.565527:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00010000:00000001:3.0:1788575285.013465:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575285.013470:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575285.013474:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575285.013475:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575285.013477:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575285.013478:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575285.013533:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575285.013536:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575285.013538:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575285.141477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575285.141483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575285.141485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575285.141489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575285.141491:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575285.141494:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575285.141496:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575285.141503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575285.141504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575286.037499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575286.037508:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575286.037510:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575286.037513:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575286.037524:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575286.037527:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575286.037529:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575286.037531:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575286.037535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575286.037536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575286.037540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575286.037541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575286.037600:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575286.037603:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575286.037605:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575286.165486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575286.165492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575286.165494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575286.165498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575286.165501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575286.165503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575286.165505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575286.165507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575286.165509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575287.061493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575287.061500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575287.061505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575287.061506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575287.061510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575287.061511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575287.061588:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575287.061591:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575287.061594:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575287.189488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575287.189494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575287.189495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575287.189498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575287.189501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575287.189504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575287.189506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575287.189507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575287.189509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575288.085535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575288.085541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575288.085545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575288.085546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575288.085548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575288.085549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575288.085611:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575288.085616:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575288.085619:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575288.213542:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575288.213551:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575288.213553:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575288.213558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575288.213561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575288.213565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575288.213567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575288.213569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575288.213571:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575288.725528:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575288.725537:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575288.725540:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575288.725543:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575288.725547:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575288.725553:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12219
00000400:00000200:2.0:1788575288.725560:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575288.725565:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575288.725568:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575288.725571:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725574:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575288.725576:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725578:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575288.725581:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575288.725582:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575288.725584:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575288.725587:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575288.725588:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575288.725590:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575288.725592:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725593:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575288.725595:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725596:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575288.725599:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575288.725601:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725603:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575288.725605:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575288.725608:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575288.725610:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575288.725612:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575288.725613:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575288.725622:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00e00.
00000020:00000040:2.0:1788575288.725624:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575288.725627:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575288.725628:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575288.725631:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575288.725633:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575288.725637:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575288.725643:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:2.0:1788575288.725649:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575288.725656:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575288.725665:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575288.725668:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575288.725671:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5f665f0] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575288.725690:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575288.725693:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575288.725695:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575288.725697:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575288.725698:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575288.725700:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575288.725702:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12219
00000400:00000200:2.0:1788575288.725705:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575288.725707:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575288.725709:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575288.725711:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725713:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575288.725715:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725716:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575288.725718:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575288.725719:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575288.725720:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575288.725722:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575288.725724:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575288.725726:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575288.725728:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725730:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575288.725730:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1788575288.725733:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575288.725736:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575288.725737:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575288.725741:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575288.725741:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1788575288.725746:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725750:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575288.725752:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1788575288.725752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575288.725759:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575288.725764:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575288.725766:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575288.725766:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575288.725769:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1788575288.725771:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575288.725773:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:2.0:1788575288.725775:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575288.725777:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575288.725778:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575288.725784:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01c00.
02000000:00000001:3.0:1788575288.725784:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575288.725790:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00100000:3.0:1788575288.725792:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393198208:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1788575288.725793:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575288.725796:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575288.725798:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1788575288.725799:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575288.725801:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575288.725802:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575288.725803:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575288.725806:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575288.725808:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575288.725809:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788575288.725812:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575288.725816:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9400.
00000100:00100000:2.0:1788575288.725820:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1788575288.725821:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575288.725831:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575288.725832:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575288.725835:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575288.725836:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000040:2.0:1788575288.725841:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000de3acaa1] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788575288.725845:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000001:2.0:1788575288.725849:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575288.725853:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1788575288.725854:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393198208, portal 25
00000100:00080000:2.0:1788575288.725856:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575288.725859:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575288.725860:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1788575288.725862:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00080000:2.0:1788575288.725863:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575288.725868:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:3.0:1788575288.725872:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575288.725882:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12219
00000100:00000001:3.0:1788575288.725883:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575288.725889:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575288.725891:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575288.725891:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575288.725896:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393198208, offset 0
00000400:00000200:3.0:1788575288.725903:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575288.725912:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575288.725915:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575288.725919:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575288.725920:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1788575288.725927:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575288.725929:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788575288.725932:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575288.725936:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:2.0:1788575288.725937:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725941:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575288.725943:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1788575288.725945:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575288.725949:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:3.0:1788575288.725949:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:2.0:1788575288.725954:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575288.725958:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575288.725958:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330804:330804:256:4294967295] 192.168.202.110@tcp LPNI seq info [330761:330761:8:4294967295]
00000020:00000001:2.0:1788575288.725962:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575288.725966:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575288.725967:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575288.725971:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725973:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575288.725978:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575288.725984:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575288.725985:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1788575288.725990:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:3.0:1788575288.725997:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575288.725998:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575288.726004:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575288.726007:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:3.0:1788575288.726007:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
02000000:00000001:2.0:1788575288.726012:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575288.726017:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575288.726021:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575288.726022:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575288.726027:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:3.0:1788575288.726029:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000008:00000001:2.0:1788575288.726031:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575288.726037:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575288.726041:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00a80.
00000800:00000200:3.1:1788575288.726041:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1788575288.726047:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:3.1:1788575288.726048:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:2.0:1788575288.726053:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575288.726056:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1788575288.726057:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575288.726060:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:3.0:1788575288.726062:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
02000000:00000001:2.0:1788575288.726064:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575288.726068:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575288.726070:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:2.0:1788575288.726073:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000400:00000200:3.0:1788575288.726080:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1788575288.726081:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575288.726087:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000100:00100000:2.0:1788575288.726091:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575288.726093:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575288.726102:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:2.0:1788575288.726104:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575288.726110:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575288.726112:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575288.726117:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000389a8694] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726132:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1788575288.726133:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575288.726137:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575288.726140:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575288.726141:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575288.726148:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12229)
00000100:00000001:3.0:1788575288.726164:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575288.726166:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575288.726173:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575288.726177:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575288.726179:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306893241
00000100:00000010:3.0:1788575288.726184:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575288.726190:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575288.726195:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306893241
00000100:00000010:3.0:1788575288.726197:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575288.726199:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575288.726201:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575288.726203:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575288.726205:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575288.726209:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575288/real 1788575288]  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575288.726216:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575288.726217:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575288.726219:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575288.726221:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575288.726222:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575288.726224:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575288.726226:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575288.726232:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726237:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575288.726238:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726240:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575288.726242:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575288.726245:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726253:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575288.726258:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726266:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575288.726269:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726271:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575288.726273:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726276:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575288.726278:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575288.726281:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575288.726285:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575288.726288:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575288.726293:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575288.726299:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393198208:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575288.726303:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726304:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575288.726306:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393198208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726311:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575288.726313:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575288.726315:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575288.726317:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575288.726319:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575288.726320:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575288.726322:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575288.726323:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575288.726327:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575288.726329:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575288.726330:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:3.0:1788575288.726332:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575288.726333:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726335:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726341:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575288.726343:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575288.726344:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575288.726345:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575288.726347:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575288.726349:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726354:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575288.726355:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575288.726356:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575288.726358:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575288.726359:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575288.726361:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198336:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575288.726363:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575288.726364:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575288.726366:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575288.726367:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575288.726369:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575288.726371:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575288.726373:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575288.726375:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575288.726378:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575288.726380:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393198336, portal 10
00000100:00000001:3.0:1788575288.726381:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575288.726383:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:3.0:1788575288.726388:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726421:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575288.726423:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575288.726425:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575288.726427:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393198336, offset 0
00000400:00000200:3.0:1788575288.726430:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575288.726433:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575288.726436:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575288.726439:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575288.726441:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575288.726442:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575288.726444:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575288.726446:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330805:330805:256:4294967295] 192.168.202.110@tcp LPNI seq info [330762:330762:8:4294967295]
00000400:00000200:3.0:1788575288.726449:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575288.726454:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575288.726457:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575288.726459:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575288.726462:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575288.726464:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575288.726467:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575288.726470:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575288.726473:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575288.726474:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575288.726476:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575288.726478:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575288.726481:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575288.726482:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575288.726484:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575288.726486:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726509:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575288.726512:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726516:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575288.726518:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575288.726521:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575288.726524:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575288.726526:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575288.726528:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306892327
00000100:00000001:3.0:1788575288.726530:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306893241
00000100:00000001:3.0:1788575288.726532:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306892327
00000100:00000001:3.0:1788575288.726533:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306893241
00000100:00000001:3.0:1788575288.726535:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575288.726536:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575288.726538:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575288.726540:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575288/real 1788575288]  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575288.726545:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575288.726546:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575288.726547:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575288.726548:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575288.726549:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575288.726551:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575288.726553:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575288.726558:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726562:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575288.726563:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726565:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575288.726565:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575288.726568:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726572:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575288.726574:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726579:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575288.726581:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726582:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575288.726583:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726584:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575288.726586:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575288.726587:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575288.726589:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575288.726591:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575288.726593:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575288.726598:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198336:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575288.726600:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575288.726602:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726607:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575288.726608:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575288.726610:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575288.726611:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575288.726612:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575288.726614:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575288.726616:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575288.726618:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575288.726619:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575288.726620:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575288.726623:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8000.
02000000:00000001:3.0:1788575288.726624:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575288.726626:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575288.726628:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575288.726630:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575288.726632:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393198464, portal 4
00000100:00000001:3.0:1788575288.726634:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575288.726635:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:3.0:1788575288.726639:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726643:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575288.726644:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575288.726646:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575288.726647:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393198464, offset 0
00000400:00000200:3.0:1788575288.726650:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575288.726652:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575288.726654:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575288.726656:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575288.726658:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575288.726660:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575288.726662:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575288.726664:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330806:330806:256:4294967295] 192.168.202.110@tcp LPNI seq info [330763:330763:8:4294967295]
00000400:00000200:3.0:1788575288.726668:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575288.726675:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575288.726680:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575288.726683:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575288.726687:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575288.726690:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575288.726694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575288.726697:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575288.726701:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575288.726703:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575288.726706:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575288.726710:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575288.726714:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575288.726716:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575288.726719:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575288.726723:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726729:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575288.726731:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726735:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575288.726737:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575288.726739:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575288.726742:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575288.726744:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575288.726745:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306892473
00000100:00000001:3.0:1788575288.726747:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306893241
00000100:00000001:3.0:1788575288.726749:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306892473
00000100:00000001:3.0:1788575288.726750:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306893241
00000100:00000001:3.0:1788575288.726752:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575288.726753:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575288.726754:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575288.726756:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575288/real 1788575288]  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575288.726761:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575288.726762:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575288.726763:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575288.726765:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575288.726766:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575288.726767:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575288.726769:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575288.726775:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726783:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575288.726784:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726785:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575288.726786:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575288.726788:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726792:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575288.726795:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726799:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575288.726801:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575288.726803:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726804:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575288.726805:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575288.726807:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575288.726809:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575288.726810:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575288.726812:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575288.726818:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575288.726820:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726821:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575288.726823:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393198336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726827:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575288.726828:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575288.726830:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575288.726831:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575288.726832:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575288.726833:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575288.726835:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575288.726836:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575288.726838:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575288.726838:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575288.726839:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:3.0:1788575288.726841:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575288.726842:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726843:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575288.726845:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393198464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575288.726849:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575288.726850:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575288.726851:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8000.
02000000:00000001:3.0:1788575288.726852:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575288.726853:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575288.726854:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575288.726855:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575288.726856:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575288.726857:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575288.726857:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575288.726858:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575288.726859:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575288.726860:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726861:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575288.726864:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575288.726865:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575288.726866:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575288.726867:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575289.109532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575289.109540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575289.109547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575289.109548:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575289.109552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575289.109553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575289.109606:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575289.109611:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575289.109615:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575289.237534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575289.237542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575289.237544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575289.237550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575289.237553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575289.237557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575289.237560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575289.237563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575289.237566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575289.685506:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306893241, at 4306893248
00000100:00000040:1.0:1788575289.685506:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306893241, at 4306893248
00000100:00000010:0.0:1788575289.685521:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:1.0:1788575289.685524:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000800:00000200:0.0:1788575289.749528:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788575289.749535:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788575289.749540:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788575289.749715:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1788575289.749729:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788575289.749784:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788575289.749790:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575289.749793:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575290.133516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575290.133522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575290.133525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575290.133526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575290.133529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575290.133530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575290.133592:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575290.133596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575290.133600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575290.261536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575290.261545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575290.261547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575290.261552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575290.261556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575290.261560:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575290.261563:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575290.261566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575290.261569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575291.157522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575291.157528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575291.157534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575291.157536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575291.157539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575291.157541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575291.157600:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575291.157604:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575291.157606:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575291.285511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575291.285519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575291.285522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575291.285527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575291.285531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575291.285534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575291.285537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575291.285539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575291.285542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575292.181493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575292.181499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575292.181506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575292.181508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575292.181511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575292.181513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575292.181541:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575292.181545:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575292.181550:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575292.309540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575292.309549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575292.309551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575292.309556:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575292.309560:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575292.309564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575292.309567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575292.309570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575292.309573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575293.205536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575293.205543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575293.205549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575293.205551:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575293.205555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575293.205556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575293.205609:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575293.205614:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575293.205617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575293.333489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575293.333497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575293.333511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575293.333518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575293.333523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575293.333527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575293.333530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575293.333533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575293.333536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575293.845520:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575293.845529:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575293.845531:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575293.845534:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575293.845538:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575293.845544:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12224
00000400:00000200:3.0:1788575293.845551:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575293.845556:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575293.845558:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575293.845561:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575293.845564:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575293.845566:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575293.845568:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575293.845570:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575293.845572:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575293.845574:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575293.845577:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575293.845578:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575293.845580:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575293.845582:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575293.845583:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575293.845585:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575293.845586:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575293.845589:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575293.845591:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575293.845593:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575293.845595:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575293.845598:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575293.845600:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575293.845601:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575293.845603:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575293.845611:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788575293.845613:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575293.845616:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575293.845617:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575293.845619:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575293.845622:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575293.845625:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575293.845631:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575293.845636:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575293.845643:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575293.845654:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575293.845657:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575293.845661:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575293.845678:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575293.845681:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575293.845684:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575293.845685:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575293.845686:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575293.845688:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575293.845690:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12224
00000400:00000200:3.0:1788575293.845693:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575293.845695:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575293.845697:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575293.845700:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575293.845702:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575293.845705:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575293.845706:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575293.845708:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575293.845711:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575293.845714:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575293.845715:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788575293.845719:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575293.845723:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575293.845725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575293.845727:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575293.845734:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575293.845740:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575293.845749:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575293.845751:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575293.845754:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575293.845755:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575293.845757:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1788575293.845759:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575293.845761:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575293.845763:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575293.845764:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575293.845769:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00100000:2.0:1788575293.845772:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393198592:192.168.202.110@tcp:250:kworker.0
00000100:00080000:3.0:1788575293.845775:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575293.845778:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575293.845779:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575293.845782:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575293.845783:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575293.845785:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575293.845787:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1788575293.845788:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:2.0:1788575293.845792:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575293.845792:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575293.845793:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575293.845797:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575293.845799:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575293.845803:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575293.845804:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575293.845806:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575293.845809:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000100:00000010:3.0:1788575293.845812:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000400:00000010:2.0:1788575293.845816:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000020:00000040:3.0:1788575293.845817:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000200:2.0:1788575293.845823:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393198592, portal 25
02000000:00000001:3.0:1788575293.845823:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575293.845825:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.845827:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575293.845829:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1788575293.845830:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
02000000:00000001:3.0:1788575293.845834:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:2.0:1788575293.845837:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575293.845839:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.845847:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000010:3.0:1788575293.845848:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000040:2.0:1788575293.845851:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575293.845855:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000001:3.0:1788575293.845856:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575293.845858:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393198592, offset 0
00000100:00100000:3.0:1788575293.845864:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575293.845866:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575293.845874:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575293.845877:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575293.845879:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575293.845885:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1788575293.845887:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575293.845891:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575293.845897:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575293.845903:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575293.845907:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575293.845909:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575293.845913:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330807:330807:256:4294967295] 192.168.202.110@tcp LPNI seq info [330764:330764:8:4294967295]
00000100:00080000:3.0:1788575293.845916:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575293.845919:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575293.845924:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575293.845926:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575293.845928:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575293.845934:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575293.845936:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1788575293.845939:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12224
00000800:00000200:2.0:1788575293.845947:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575293.845948:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575293.845954:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1788575293.845955:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000100:00080000:3.0:1788575293.845956:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575293.845961:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575293.845964:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1788575293.845965:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575293.845968:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.0:1788575293.845971:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000100:00000001:3.0:1788575293.845972:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575293.845977:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575293.845979:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575293.845983:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.1:1788575293.845983:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1788575293.845986:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:2.1:1788575293.845987:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000020:00000001:3.0:1788575293.845989:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575293.845992:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000100:2.0:1788575293.845996:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575293.845999:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575293.846000:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000100:00000001:3.0:1788575293.846003:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575293.846006:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1788575293.846008:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575293.846014:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1788575293.846014:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575293.846018:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000020:00000040:3.0:1788575293.846019:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:2.0:1788575293.846022:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575293.846026:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00080000:3.0:1788575293.846027:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:2.0:1788575293.846033:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575293.846034:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575293.846039:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575293.846045:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575293.846046:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1788575293.846051:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575293.846052:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575293.846056:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575293.846060:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575293.846062:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788575293.846063:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846065:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575293.846066:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846070:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575293.846070:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575293.846076:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575293.846078:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575293.846080:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575293.846082:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306898361
02000000:00000001:3.0:1788575293.846083:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575293.846088:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575293.846089:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000010:2.0:1788575293.846090:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ce40.
02000000:00000001:3.0:1788575293.846091:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575293.846097:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:2.0:1788575293.846099:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575293.846101:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846105:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306898361
00000100:00100000:3.0:1788575293.846108:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:2.0:1788575293.846110:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c240.
00000100:00000001:2.0:1788575293.846116:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575293.846119:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575293.846120:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575293.846123:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846124:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846128:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1788575293.846129:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575293.846135:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575293/real 1788575293]  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575293.846139:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575293.846143:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575293.846150:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000001:2.0:1788575293.846151:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575293.846153:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1788575293.846154:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12235)
00000400:00000010:2.0:1788575293.846155:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575293.846160:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575293.846161:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575293.846163:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575293.846166:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575293.846173:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846177:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575293.846178:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846180:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575293.846182:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575293.846184:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846189:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575293.846192:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846197:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575293.846199:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846200:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575293.846200:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846202:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575293.846203:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575293.846205:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575293.846208:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575293.846210:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575293.846212:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575293.846218:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393198592:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575293.846221:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846222:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575293.846224:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393198592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846228:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575293.846230:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575293.846232:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575293.846234:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575293.846235:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575293.846237:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575293.846238:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575293.846240:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575293.846242:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575293.846244:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575293.846245:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575293.846247:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575293.846248:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846249:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846255:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575293.846256:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846258:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575293.846259:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575293.846260:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575293.846262:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846266:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575293.846268:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575293.846269:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575293.846271:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575293.846272:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575293.846274:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575293.846276:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575293.846278:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575293.846279:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575293.846280:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575293.846282:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575293.846283:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575293.846285:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575293.846287:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575293.846289:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575293.846291:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393198720, portal 10
00000100:00000001:2.0:1788575293.846292:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575293.846293:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:2.0:1788575293.846298:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846302:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575293.846304:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575293.846305:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575293.846306:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393198720, offset 0
00000400:00000200:2.0:1788575293.846309:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575293.846311:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575293.846313:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575293.846316:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575293.846317:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575293.846319:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575293.846321:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575293.846323:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330808:330808:256:4294967295] 192.168.202.110@tcp LPNI seq info [330765:330765:8:4294967295]
00000400:00000200:2.0:1788575293.846350:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575293.846355:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575293.846358:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575293.846361:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575293.846364:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575293.846366:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788575293.846369:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575293.846370:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:2.0:1788575293.846372:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575293.846374:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575293.846376:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575293.846378:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575293.846380:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575293.846381:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575293.846383:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575293.846386:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846407:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575293.846410:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846414:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846416:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575293.846418:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575293.846420:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575293.846423:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575293.846424:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306897447
00000100:00000001:2.0:1788575293.846426:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306898361
00000100:00000001:2.0:1788575293.846427:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306897447
00000100:00000001:2.0:1788575293.846429:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306898361
00000100:00000001:2.0:1788575293.846430:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575293.846432:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846433:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575293.846435:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575293/real 1788575293]  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575293.846445:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575293.846446:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575293.846447:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575293.846448:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575293.846449:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575293.846451:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575293.846453:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575293.846458:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846462:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575293.846463:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846464:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575293.846465:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575293.846467:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846471:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575293.846473:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846477:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575293.846479:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846480:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575293.846481:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846482:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575293.846484:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575293.846485:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575293.846487:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575293.846489:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575293.846492:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575293.846497:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575293.846499:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575293.846501:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846505:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575293.846506:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575293.846507:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575293.846508:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575293.846510:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575293.846511:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198848:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575293.846513:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575293.846514:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575293.846516:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575293.846517:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575293.846520:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575293.846521:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575293.846523:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575293.846525:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575293.846527:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575293.846529:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393198848, portal 4
00000100:00000001:2.0:1788575293.846531:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575293.846532:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788575293.846535:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846540:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575293.846541:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575293.846542:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575293.846544:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393198848, offset 0
00000400:00000200:2.0:1788575293.846547:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575293.846549:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575293.846551:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575293.846553:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575293.846555:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575293.846556:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575293.846558:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575293.846560:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330809:330809:256:4294967295] 192.168.202.110@tcp LPNI seq info [330766:330766:8:4294967295]
00000400:00000200:2.0:1788575293.846563:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575293.846567:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575293.846569:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575293.846571:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575293.846574:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575293.846575:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788575293.846578:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575293.846580:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:2.0:1788575293.846582:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575293.846583:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575293.846585:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575293.846588:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575293.846589:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575293.846591:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575293.846592:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575293.846594:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846599:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575293.846601:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846605:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846606:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575293.846608:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575293.846609:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575293.846611:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575293.846613:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306897593
00000100:00000001:2.0:1788575293.846614:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306898361
00000100:00000001:2.0:1788575293.846616:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306897593
00000100:00000001:2.0:1788575293.846617:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306898361
00000100:00000001:2.0:1788575293.846618:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575293.846620:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846621:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575293.846623:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575293/real 1788575293]  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575293.846627:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575293.846628:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575293.846629:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575293.846631:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575293.846632:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575293.846633:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575293.846635:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575293.846641:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846645:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575293.846645:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846646:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575293.846647:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575293.846649:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846653:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575293.846656:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846660:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575293.846662:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846663:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575293.846664:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846665:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575293.846666:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575293.846668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575293.846669:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575293.846671:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575293.846673:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575293.846678:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393198848:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575293.846680:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846681:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575293.846683:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393198720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846687:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575293.846687:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575293.846689:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575293.846690:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575293.846691:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575293.846692:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575293.846693:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575293.846694:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575293.846696:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575293.846697:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575293.846698:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575293.846699:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575293.846700:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846701:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575293.846703:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393198848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575293.846712:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575293.846713:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575293.846714:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575293.846715:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575293.846716:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575293.846717:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575293.846718:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575293.846719:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575293.846720:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575293.846721:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575293.846721:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575293.846723:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575293.846723:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846724:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575293.846726:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575293.846727:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575293.846728:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575293.846729:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575294.229520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575294.229525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575294.229528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575294.229531:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575294.229533:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575294.229535:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575294.229538:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575294.229539:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575294.229541:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575294.229543:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575294.229545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575294.229546:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575294.229547:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575294.229548:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575294.229549:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575294.229550:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575294.229551:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575294.229552:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575294.229608:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575294.229612:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575294.229614:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575294.357495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575294.357501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575294.357503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575294.357507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575294.357510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575294.357513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575294.357515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575294.357518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575294.357520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575294.805511:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306898361, at 4306898368
00000100:00000040:1.0:1788575294.805511:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306898361, at 4306898368
00000100:00000010:0.0:1788575294.805523:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000100:00000010:1.0:1788575294.805528:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00010000:00000001:3.0:1788575295.253526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575295.253534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575295.253540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575295.253541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575295.253545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575295.253547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575295.253608:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575295.253611:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575295.253614:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575295.381473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575295.381478:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575295.381480:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575295.381485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575295.381487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575295.381490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575295.381492:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575295.381494:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575295.381497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575296.277508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575296.277514:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575296.277516:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575296.277519:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575296.277523:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575296.277524:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575296.277526:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575296.277528:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575296.277530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575296.277531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575296.277533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575296.277534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575296.277587:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575296.277590:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575296.277593:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575296.405522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575296.405530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575296.405533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575296.405539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575296.405543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575296.405547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575296.405549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575296.405552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575296.405565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575297.301505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575297.301510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575297.301513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575297.301514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575297.301516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575297.301517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575297.301583:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575297.301586:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575297.301590:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575297.429475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575297.429481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575297.429482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575297.429486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575297.429489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575297.429492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575297.429494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575297.429496:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575297.429498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575298.325473:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575298.325479:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575298.325483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575298.325484:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575298.325487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575298.325487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575298.325547:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575298.325552:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575298.325555:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575298.453460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575298.453467:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575298.453470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575298.453475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575298.453479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575298.453483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.453486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575298.453489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.453492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575298.965496:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575298.965505:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575298.965507:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575298.965510:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575298.965515:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575298.965520:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12230
00000400:00000200:3.0:1788575298.965527:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575298.965533:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575298.965535:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575298.965538:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575298.965540:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575298.965542:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575298.965544:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575298.965547:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575298.965548:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575298.965551:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575298.965553:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575298.965555:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575298.965556:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575298.965558:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575298.965559:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575298.965561:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575298.965562:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575298.965565:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575298.965567:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575298.965569:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575298.965571:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575298.965574:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575298.965576:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575298.965577:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575298.965579:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575298.965587:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575298.965589:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575298.965592:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575298.965593:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575298.965595:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575298.965597:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575298.965601:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575298.965607:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
00000100:00000001:3.0:1788575298.965613:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575298.965620:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575298.965628:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575298.965632:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575298.965635:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575298.965659:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575298.965661:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575298.965664:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575298.965665:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575298.965666:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575298.965669:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575298.965671:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12230
00000400:00000200:3.0:1788575298.965674:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575298.965676:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575298.965678:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575298.965681:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575298.965683:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575298.965684:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575298.965686:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575298.965687:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575298.965688:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575298.965689:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575298.965690:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575298.965694:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575298.965695:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575298.965696:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575298.965700:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575298.965700:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575298.965706:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575298.965709:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575298.965713:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575298.965714:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575298.965720:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575298.965728:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575298.965732:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575298.965733:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575298.965738:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:3.0:1788575298.965740:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575298.965743:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575298.965745:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575298.965749:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575298.965750:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575298.965758:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1788575298.965759:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393198976:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575298.965765:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.965768:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575298.965770:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575298.965774:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:3.0:1788575298.965776:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575298.965779:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788575298.965783:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575298.965785:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575298.965787:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575298.965793:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
00000100:00000010:3.0:1788575298.965795:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
02000000:00000001:2.0:1788575298.965797:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575298.965801:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575298.965806:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575298.965807:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575298.965809:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575298.965814:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000100:00000001:3.0:1788575298.965814:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575298.965817:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575298.965821:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575298.965827:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa000.
00000400:00000010:2.0:1788575298.965837:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000001:3.0:1788575298.965842:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575298.965847:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393198976, portal 25
00000100:00000001:2.0:1788575298.965852:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00100000:3.0:1788575298.965854:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575298.965856:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:2.0:1788575298.965868:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575298.965870:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575298.965875:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575298.965882:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.965890:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575298.965898:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575298.965903:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575298.965908:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393198976, offset 0
00000100:00000001:3.0:1788575298.965914:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575298.965919:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575298.965921:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575298.965927:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:2.0:1788575298.965930:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575298.965931:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575298.965936:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575298.965938:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788575298.965942:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575298.965945:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1788575298.965949:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12230
00000400:00000200:2.0:1788575298.965952:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575298.965958:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575298.965960:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575298.965963:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330810:330810:256:4294967295] 192.168.202.110@tcp LPNI seq info [330767:330767:8:4294967295]
00000400:00000200:2.0:1788575298.965966:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575298.965972:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575298.965979:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1788575298.965984:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:2.0:1788575298.965989:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575298.965991:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000010:2.0:1788575298.965994:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000100:00000040:3.0:1788575298.965999:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.0:1788575298.966005:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575298.966006:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1788575298.966012:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00000040:3.0:1788575298.966013:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575298.966020:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575298.966021:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.1:1788575298.966026:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:3.0:1788575298.966027:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000100:2.0:1788575298.966031:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575298.966031:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000010:2.0:1788575298.966035:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000020:00000001:3.0:1788575298.966037:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:2.0:1788575298.966041:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:3.0:1788575298.966042:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575298.966048:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575298.966049:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:3.0:1788575298.966055:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575298.966056:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575298.966060:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000040:3.0:1788575298.966063:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575298.966065:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:3.0:1788575298.966069:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:2.0:1788575298.966072:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575298.966076:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575298.966084:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575298.966088:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788575298.966093:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1788575298.966093:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:2.0:1788575298.966099:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575298.966101:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575298.966108:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575298.966113:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966115:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:3.0:1788575298.966115:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575298.966121:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966122:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575298.966127:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788575298.966129:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:3.0:1788575298.966132:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575298.966133:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306903481
00000008:00000001:3.0:1788575298.966139:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575298.966140:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd20.
00000100:00000001:3.0:1788575298.966145:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966149:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000010:3.0:1788575298.966152:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575298.966155:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306903481
00000020:00000040:3.0:1788575298.966158:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000010:2.0:1788575298.966160:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c360.
00000100:00000001:2.0:1788575298.966180:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575298.966183:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575298.966185:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966187:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575298.966191:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575298/real 1788575298]  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575298.966192:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575298.966196:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575298.966201:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575298.966204:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000001:2.0:1788575298.966205:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:3.0:1788575298.966208:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1788575298.966209:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575298.966214:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575298.966219:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
02000000:00000010:3.0:1788575298.966219:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000400:00000010:2.0:1788575298.966223:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:3.0:1788575298.966226:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966228:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575298.966241:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575298.966242:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575298.966269:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1788575298.966270:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575298.966273:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966282:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:3.0:1788575298.966284:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966285:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966290:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575298.966293:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575298.966301:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575298.966303:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575298.966309:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575298.966311:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575298.966318:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575298.966319:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575298.966327:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12240)
00000100:00000001:2.0:1788575298.966332:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575298.966338:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966339:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575298.966341:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966344:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575298.966346:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575298.966356:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575298.966361:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575298.966366:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575298.966371:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575298.966380:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393198976:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575298.966386:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966388:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575298.966422:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393198976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966428:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575298.966431:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575298.966433:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575298.966436:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575298.966437:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575298.966438:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575298.966440:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575298.966442:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:2.0:1788575298.966445:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575298.966447:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575298.966448:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575298.966451:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575298.966451:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966453:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966459:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575298.966461:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966462:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575298.966464:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575298.966465:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575298.966468:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966473:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575298.966474:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575298.966475:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575298.966477:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575298.966479:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575298.966480:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199104:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575298.966483:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575298.966484:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575298.966486:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575298.966487:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575298.966490:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575298.966491:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575298.966494:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575298.966496:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575298.966499:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575298.966501:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393199104, portal 10
00000100:00000001:2.0:1788575298.966503:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575298.966503:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:2.0:1788575298.966508:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966513:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575298.966516:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575298.966518:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575298.966520:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393199104, offset 0
00000400:00000200:2.0:1788575298.966525:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575298.966529:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575298.966533:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575298.966536:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575298.966539:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575298.966541:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575298.966544:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575298.966547:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330811:330811:256:4294967295] 192.168.202.110@tcp LPNI seq info [330768:330768:8:4294967295]
00000400:00000200:2.0:1788575298.966552:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575298.966559:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575298.966564:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575298.966568:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575298.966572:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575298.966575:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788575298.966578:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575298.966580:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:2.0:1788575298.966583:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575298.966584:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575298.966587:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575298.966590:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575298.966592:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575298.966593:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575298.966595:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575298.966598:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966603:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575298.966605:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966610:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966611:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575298.966613:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575298.966616:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575298.966619:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575298.966620:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306902567
00000100:00000001:2.0:1788575298.966623:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306903481
00000100:00000001:2.0:1788575298.966624:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306902567
00000100:00000001:2.0:1788575298.966626:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306903481
00000100:00000001:2.0:1788575298.966627:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575298.966628:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966630:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575298.966632:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575298/real 1788575298]  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575298.966638:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575298.966638:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575298.966640:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575298.966641:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575298.966642:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575298.966643:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575298.966646:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575298.966651:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966655:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575298.966656:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966657:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575298.966658:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575298.966660:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966664:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575298.966667:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966671:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575298.966672:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966673:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575298.966674:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966675:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575298.966677:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575298.966678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575298.966680:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575298.966682:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575298.966685:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575298.966690:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199104:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575298.966692:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575298.966694:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966703:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575298.966704:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575298.966706:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575298.966707:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575298.966709:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575298.966710:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199232:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575298.966712:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575298.966714:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575298.966715:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575298.966716:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575298.966718:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575298.966719:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575298.966721:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575298.966723:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575298.966725:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575298.966727:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393199232, portal 4
00000100:00000001:2.0:1788575298.966728:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575298.966729:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788575298.966733:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966737:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575298.966739:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575298.966740:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575298.966742:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393199232, offset 0
00000400:00000200:2.0:1788575298.966744:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575298.966746:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575298.966749:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575298.966751:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575298.966752:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575298.966754:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575298.966756:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575298.966758:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330812:330812:256:4294967295] 192.168.202.110@tcp LPNI seq info [330769:330769:8:4294967295]
00000400:00000200:2.0:1788575298.966761:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575298.966765:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575298.966768:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575298.966770:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575298.966773:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575298.966774:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788575298.966777:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575298.966779:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:2.0:1788575298.966782:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575298.966783:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575298.966785:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575298.966787:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575298.966789:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575298.966790:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575298.966791:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575298.966794:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966799:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575298.966801:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966805:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966806:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575298.966808:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575298.966809:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575298.966811:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575298.966812:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306902713
00000100:00000001:2.0:1788575298.966814:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306903481
00000100:00000001:2.0:1788575298.966815:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306902713
00000100:00000001:2.0:1788575298.966817:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306903481
00000100:00000001:2.0:1788575298.966818:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575298.966820:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966821:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575298.966824:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575298/real 1788575298]  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575298.966829:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575298.966830:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575298.966831:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575298.966832:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575298.966833:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575298.966834:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575298.966836:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575298.966841:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966845:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575298.966846:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966847:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575298.966848:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575298.966850:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966854:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575298.966856:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966860:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575298.966862:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966862:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575298.966863:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966865:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575298.966866:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575298.966867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575298.966870:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575298.966871:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575298.966874:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575298.966879:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199232:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575298.966881:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966882:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575298.966884:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393199104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966888:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575298.966889:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575298.966890:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575298.966891:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575298.966892:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575298.966893:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575298.966895:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575298.966896:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:2.0:1788575298.966898:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575298.966899:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575298.966900:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:2.0:1788575298.966902:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575298.966902:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966904:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575298.966906:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393199232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575298.966909:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575298.966910:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575298.966911:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575298.966913:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575298.966914:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575298.966915:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575298.966916:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575298.966917:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575298.966918:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575298.966919:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575298.966920:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575298.966922:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575298.966922:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966924:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575298.966926:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575298.966927:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575298.966928:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575298.966929:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575299.349486:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575299.349493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575299.349499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575299.349500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575299.349504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575299.349505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575299.349539:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575299.349543:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575299.349547:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575299.477477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575299.477485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575299.477487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575299.477494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575299.477498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575299.477502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575299.477505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575299.477508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575299.477511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575299.925494:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306903481, at 4306903488
00000100:00000040:0.0:1788575299.925499:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306903481, at 4306903488
00000100:00000010:1.0:1788575299.925508:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000100:00000010:0.0:1788575299.925511:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00010000:00000001:3.0:1788575300.373473:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575300.373478:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575300.373482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575300.373483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575300.373485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575300.373486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575300.373551:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575300.373556:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575300.373560:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575300.501555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575300.501562:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575300.501565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575300.501571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575300.501576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575300.501580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575300.501584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575300.501587:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575300.501591:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575301.397492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575301.397497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575301.397502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575301.397503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575301.397506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575301.397507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575301.397578:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575301.397582:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575301.397586:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575301.525510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575301.525517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575301.525519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575301.525523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575301.525525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575301.525529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575301.525531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575301.525533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575301.525535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575302.421518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575302.421527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575302.421535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575302.421537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575302.421541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575302.421543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575302.421603:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575302.421608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575302.421612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575302.549516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575302.549524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575302.549527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575302.549533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575302.549536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575302.549541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575302.549543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575302.549546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575302.549550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575303.445490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575303.445495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575303.445499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575303.445500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575303.445502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575303.445503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575303.445561:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575303.445565:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575303.445569:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575303.573510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575303.573520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575303.573522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575303.573528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575303.573531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575303.573536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575303.573538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575303.573541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575303.573543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575304.085475:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575304.085484:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575304.085487:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575304.085490:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575304.085494:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575304.085500:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12235
00000400:00000200:3.0:1788575304.085508:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575304.085514:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575304.085517:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575304.085520:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085523:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575304.085525:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085527:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575304.085530:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575304.085531:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575304.085533:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575304.085536:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575304.085537:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575304.085540:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575304.085542:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085544:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575304.085545:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085547:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575304.085549:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575304.085551:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085553:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575304.085555:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575304.085558:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575304.085560:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575304.085562:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575304.085564:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575304.085571:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575304.085574:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575304.085576:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575304.085578:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575304.085580:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575304.085582:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575304.085586:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575304.085593:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575304.085598:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575304.085605:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575304.085614:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575304.085617:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575304.085620:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575304.085637:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575304.085639:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575304.085642:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575304.085643:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575304.085645:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575304.085647:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575304.085649:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12235
00000400:00000200:3.0:1788575304.085651:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575304.085653:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575304.085655:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575304.085657:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085659:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575304.085661:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085663:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575304.085664:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575304.085666:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575304.085667:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575304.085668:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575304.085670:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575304.085670:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575304.085675:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575304.085676:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575304.085678:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575304.085680:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575304.085681:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575304.085685:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085688:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:2.0:1788575304.085690:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575304.085690:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575304.085696:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085702:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575304.085706:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575304.085709:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575304.085712:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575304.085713:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575304.085716:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575304.085716:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575304.085720:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575304.085721:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:3.0:1788575304.085724:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575304.085725:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788575304.085731:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575304.085732:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393199360:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575304.085737:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575304.085738:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575304.085742:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:3.0:1788575304.085744:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
02000000:00000001:2.0:1788575304.085745:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575304.085750:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1788575304.085750:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000010:2.0:1788575304.085756:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575304.085756:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575304.085759:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575304.085760:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575304.085765:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575304.085768:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575304.085770:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575304.085773:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575304.085775:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
02000000:00000010:3.0:1788575304.085780:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000400:00000010:2.0:1788575304.085783:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000001:3.0:1788575304.085787:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575304.085793:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393199360, portal 25
00000100:00100000:3.0:1788575304.085794:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575304.085800:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575304.085805:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000001:3.0:1788575304.085805:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575304.085808:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575304.085812:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575304.085819:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575304.085822:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575304.085826:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575304.085833:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575304.085835:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575304.085838:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575304.085840:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575304.085841:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:2.0:1788575304.085846:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575304.085847:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12235
00000400:00000010:2.0:1788575304.085851:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000400:00000200:3.0:1788575304.085853:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:2.0:1788575304.085857:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393199360, offset 0
00000400:00000200:3.0:1788575304.085857:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575304.085859:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575304.085863:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575304.085868:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085868:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575304.085873:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085876:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575304.085877:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1788575304.085880:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1788575304.085881:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575304.085884:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575304.085886:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575304.085888:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575304.085891:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:3.0:1788575304.085892:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575304.085898:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575304.085900:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788575304.085901:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575304.085905:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575304.085907:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575304.085908:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575304.085911:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575304.085913:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1788575304.085917:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575304.085920:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575304.085922:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575304.085925:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575304.085926:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330813:330813:256:4294967295] 192.168.202.110@tcp LPNI seq info [330770:330770:8:4294967295]
02000000:00000001:3.0:1788575304.085928:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575304.085933:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575304.085935:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575304.085945:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575304.085947:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575304.085949:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575304.085951:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575304.085957:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000400:00000200:2.0:1788575304.085961:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:3.0:1788575304.085961:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575304.085965:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575304.085967:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575304.085970:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575304.085973:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:2.0:1788575304.085974:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575304.085975:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575304.085980:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575304.085983:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575304.085986:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000100:00100000:3.0:1788575304.085988:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575304.085996:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:2.0:1788575304.085997:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575304.085998:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575304.086014:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000040:3.0:1788575304.086020:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788575304.086025:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575304.086030:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:3.0:1788575304.086034:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575304.086039:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000100:2.0:1788575304.086042:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575304.086045:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000100:00000040:3.0:1788575304.086047:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575304.086051:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1788575304.086052:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12245)
00000400:00000200:2.0:1788575304.086060:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575304.086065:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575304.086067:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575304.086071:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575304.086075:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086083:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575304.086086:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575329 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086091:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575304.086092:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575304.086097:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575304.086100:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575304.086103:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306908601
00000100:00000010:2.0:1788575304.086112:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c8a0.
00000100:00000001:2.0:1788575304.086117:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575304.086123:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306908601
00000100:00000010:2.0:1788575304.086125:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c960.
00000100:00000001:2.0:1788575304.086127:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575304.086129:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575304.086131:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575304.086133:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575304.086137:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575304/real 1788575304]  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575304.086149:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575304.086150:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575304.086152:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575304.086154:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575304.086155:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575304.086157:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575304.086159:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575304.086165:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086169:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575304.086170:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086172:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575304.086173:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575304.086176:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086180:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575304.086183:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086188:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575304.086190:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086191:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575304.086192:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086193:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575304.086195:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575304.086197:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575304.086199:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575304.086201:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575304.086203:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575304.086210:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393199360:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575304.086213:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086215:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575304.086217:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393199360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086221:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575304.086223:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575304.086225:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575304.086226:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575304.086228:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575304.086229:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575304.086230:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575304.086232:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575304.086235:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575304.086236:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575304.086237:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575304.086240:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575304.086240:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086242:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086247:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575304.086249:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575304.086250:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575304.086251:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575304.086252:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575304.086255:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086260:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575304.086261:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575304.086262:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575304.086264:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575304.086265:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575304.086267:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575304.086269:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575304.086271:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575304.086272:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575304.086273:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575304.086275:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575304.086276:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575304.086279:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575304.086281:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575304.086283:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575304.086285:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393199488, portal 10
00000100:00000001:2.0:1788575304.086287:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575304.086288:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:2.0:1788575304.086293:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086297:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575304.086299:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575304.086300:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575304.086302:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393199488, offset 0
00000400:00000200:2.0:1788575304.086305:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575304.086307:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575304.086310:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575304.086312:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575304.086314:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575304.086316:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575304.086318:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575304.086320:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330814:330814:256:4294967295] 192.168.202.110@tcp LPNI seq info [330771:330771:8:4294967295]
00000400:00000200:2.0:1788575304.086323:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575304.086327:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575304.086330:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575304.086333:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575304.086335:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575304.086337:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575304.086340:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575304.086342:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1788575304.086345:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575304.086346:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575304.086348:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575304.086351:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575304.086360:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575304.086362:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575304.086363:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575304.086366:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575304.086373:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575329 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086378:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575304.086379:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575304.086381:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575304.086384:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575304.086386:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575304.086387:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306907686
00000100:00000001:2.0:1788575304.086390:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306908601
00000100:00000001:2.0:1788575304.086409:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306907687
00000100:00000001:2.0:1788575304.086411:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306908601
00000100:00000001:2.0:1788575304.086413:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575304.086414:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575304.086415:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575304.086418:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575304/real 1788575304]  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575304.086423:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575304.086424:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575304.086426:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575304.086428:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575304.086429:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575304.086430:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575304.086432:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575304.086438:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086442:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575304.086443:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086444:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575304.086445:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575304.086447:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086451:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575304.086453:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086458:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575304.086459:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086461:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575304.086461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086463:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575304.086464:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575304.086466:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575304.086468:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575304.086469:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575304.086472:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575304.086477:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575304.086480:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575304.086482:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086487:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575304.086488:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575304.086489:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575304.086490:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575304.086491:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575304.086493:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575304.086495:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575304.086496:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575304.086497:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575304.086499:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575304.086501:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575304.086502:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575304.086505:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575304.086506:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575304.086508:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575304.086510:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393199616, portal 4
00000100:00000001:2.0:1788575304.086512:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575304.086513:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:2.0:1788575304.086517:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086521:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575304.086522:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575304.086524:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000100:00000200:2.0:1788575304.086525:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393199616, offset 0
00000400:00000200:2.0:1788575304.086528:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575304.086530:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575304.086532:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575304.086534:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575304.086537:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575304.086538:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575304.086540:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575304.086542:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330815:330815:256:4294967295] 192.168.202.110@tcp LPNI seq info [330772:330772:8:4294967295]
00000400:00000200:2.0:1788575304.086545:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575304.086549:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575304.086552:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575304.086553:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575304.086556:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575304.086558:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575304.086561:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575304.086562:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1788575304.086565:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575304.086566:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:2.0:1788575304.086568:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575304.086571:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575304.086573:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:2.0:1788575304.086574:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:2.0:1788575304.086575:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575304.086578:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086583:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575304.086584:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575329 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086589:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575304.086590:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575304.086591:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575304.086593:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575304.086596:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575304.086597:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306907833
00000100:00000001:2.0:1788575304.086599:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306908601
00000100:00000001:2.0:1788575304.086600:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306907833
00000100:00000001:2.0:1788575304.086602:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306908601
00000100:00000001:2.0:1788575304.086603:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575304.086605:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575304.086606:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575304.086609:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575304/real 1788575304]  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575304.086617:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575304.086618:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575304.086620:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:2.0:1788575304.086622:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:2.0:1788575304.086624:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:2.0:1788575304.086625:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575304.086629:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575304.086638:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086645:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575304.086647:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086649:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575304.086650:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575304.086654:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086662:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575304.086665:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086670:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575304.086672:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086673:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575304.086674:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086675:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575304.086677:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575304.086678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575304.086681:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575304.086682:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575304.086685:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575304.086689:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575304.086692:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086693:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575304.086695:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393199488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086699:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575304.086700:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575304.086702:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575304.086703:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575304.086705:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575304.086706:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575304.086707:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575304.086708:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575304.086710:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575304.086711:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575304.086712:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788575304.086714:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575304.086715:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086716:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575304.086718:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393199616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575329 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575304.086722:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575304.086722:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575304.086723:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575304.086725:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575304.086726:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575304.086727:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575304.086728:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575304.086729:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575304.086730:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575304.086731:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575304.086732:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788575304.086734:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575304.086734:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086735:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575304.086738:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575304.086739:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575304.086740:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575304.086741:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575304.469470:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575304.469476:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575304.469480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575304.469482:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575304.469488:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575304.469490:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575304.469493:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575304.469494:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575304.469496:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575304.469497:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575304.469500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575304.469501:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575304.469502:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575304.469503:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575304.469504:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575304.469505:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575304.469506:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575304.469507:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575304.469547:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575304.469550:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575304.469553:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575304.597520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575304.597526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575304.597529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575304.597535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575304.597539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575304.597543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575304.597546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575304.597549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575304.597552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575305.045499:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306908601, at 4306908608
00000100:00000040:1.0:1788575305.045499:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306908601, at 4306908608
00000100:00000010:1.0:1788575305.045514:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000100:00000010:0.0:1788575305.045516:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00010000:00000001:3.0:1788575305.493493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575305.493499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575305.493503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575305.493504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575305.493507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575305.493508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575305.493555:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575305.493558:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575305.493561:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575305.621497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575305.621504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575305.621507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575305.621514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575305.621518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575305.621521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575305.621524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575305.621527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575305.621530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575306.517533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575306.517542:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575306.517545:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575306.517548:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575306.517554:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575306.517557:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575306.517559:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575306.517561:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575306.517566:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575306.517567:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575306.517571:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575306.517572:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575306.517630:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575306.517634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575306.517636:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575306.645511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575306.645520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575306.645523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575306.645529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575306.645533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575306.645537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575306.645539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575306.645542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575306.645545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575307.541494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575307.541500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575307.541517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575307.541519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575307.541523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575307.541525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575307.541591:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575307.541594:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575307.541598:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575307.669448:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575307.669454:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575307.669456:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575307.669460:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575307.669463:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575307.669466:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575307.669469:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575307.669470:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575307.669472:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575308.565580:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575308.565590:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575308.565598:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575308.565600:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575308.565604:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575308.565606:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575308.565699:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575308.565704:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575308.565708:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575308.693559:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575308.693567:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575308.693570:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575308.693575:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575308.693579:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575308.693583:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575308.693586:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575308.693588:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575308.693591:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575309.205525:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575309.205541:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575309.205545:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575309.205550:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575309.205556:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575309.205564:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12240
00000400:00000200:3.0:1788575309.205577:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575309.205585:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575309.205589:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575309.205602:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205606:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575309.205609:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205612:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575309.205617:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575309.205619:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575309.205622:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575309.205625:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575309.205627:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575309.205629:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575309.205631:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205632:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575309.205634:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205635:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575309.205637:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575309.205640:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205641:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575309.205644:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575309.205648:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575309.205650:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575309.205652:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575309.205654:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575309.205664:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575309.205667:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575309.205671:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575309.205673:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575309.205682:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575309.205685:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575309.205689:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575309.205698:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575309.205705:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575309.205714:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575309.205722:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575309.205727:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575309.205730:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575309.205752:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575309.205755:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575309.205758:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575309.205759:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575309.205760:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575309.205763:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575309.205765:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12240
00000400:00000200:3.0:1788575309.205768:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575309.205770:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575309.205772:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575309.205774:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205775:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575309.205777:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205779:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575309.205780:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575309.205782:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575309.205783:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575309.205784:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575309.205786:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575309.205787:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575309.205789:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205791:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575309.205793:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205794:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575309.205796:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575309.205798:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205799:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575309.205801:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575309.205807:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:3.0:1788575309.205808:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575309.205813:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575309.205815:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575309.205816:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575309.205819:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000008:00100000:3.0:1788575309.205824:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000040:2.0:1788575309.205828:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:3.0:1788575309.205830:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575309.205836:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575309.205852:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788575309.205853:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575309.205857:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575309.205859:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575309.205861:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575309.205864:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575309.205865:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575309.205868:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575309.205869:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575309.205872:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575309.205875:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:2.0:1788575309.205876:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393199744:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575309.205878:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575309.205884:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:2.0:1788575309.205886:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575309.205890:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575309.205892:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575309.205895:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575309.205899:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:3.0:1788575309.205899:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575309.205908:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000010:2.0:1788575309.205909:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
00000100:00000001:3.0:1788575309.205911:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575309.205914:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575309.205923:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575309.205932:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575309.205938:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000100:00000001:3.0:1788575309.205946:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575309.205951:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:2.0:1788575309.205954:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00080000:3.0:1788575309.205956:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575309.205960:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575309.205962:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:2.0:1788575309.205965:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393199744, portal 25
00000100:00000001:3.0:1788575309.205966:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:2.0:1788575309.205971:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1788575309.205971:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12240
00000100:00000001:2.0:1788575309.205976:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000400:00000200:3.0:1788575309.205980:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575309.205984:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575309.205987:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575309.205991:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575309.205992:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.205996:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575309.206001:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575309.206008:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575309.206009:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575309.206014:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:2.0:1788575309.206016:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575309.206017:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000010:2.0:1788575309.206022:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65088.
00000020:00000001:3.0:1788575309.206022:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575309.206026:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000200:2.0:1788575309.206029:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393199744, offset 0
00000020:00000001:3.0:1788575309.206030:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575309.206036:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575309.206042:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575309.206043:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.206047:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575309.206049:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.206051:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575309.206053:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575309.206056:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575309.206058:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575309.206059:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575309.206063:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575309.206066:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575309.206067:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575309.206070:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788575309.206071:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000008:00100000:3.0:1788575309.206073:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575309.206077:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575309.206078:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575309.206080:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000010:3.0:1788575309.206083:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000020:00000040:3.0:1788575309.206086:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575309.206088:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575309.206089:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575309.206090:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575309.206091:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575309.206093:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575309.206097:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575309.206101:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575309.206102:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00100000:3.0:1788575309.206107:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575309.206112:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575309.206120:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575309.206128:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575309.206130:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330816:330816:256:4294967295] 192.168.202.110@tcp LPNI seq info [330773:330773:8:4294967295]
00000100:00000001:3.0:1788575309.206132:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575309.206140:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575309.206144:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575309.206152:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575309.206154:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575309.206158:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575309.206165:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12250)
00000400:00000200:2.0:1788575309.206167:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575309.206177:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575309.206185:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133d00.
00000800:00000200:2.0:1788575309.206194:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575309.206200:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575309.206223:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575309.206229:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575309.206233:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306913721
00000100:00000010:2.0:1788575309.206240:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cb40.
00000100:00000001:2.0:1788575309.206248:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575309.206263:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306913721
00000100:00000010:2.0:1788575309.206268:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cde0.
00000100:00000001:2.0:1788575309.206272:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575309.206276:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575309.206280:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575309.206284:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575309.206288:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575309.206298:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575309.206301:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575309.206305:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575309.206307:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575309.206308:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575309.206314:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575309.206322:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575309.206325:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575309.206327:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575309.206329:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575309.206331:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575309.206335:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199872:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575309.206339:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575309.206342:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575309.206344:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575309.206346:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575309.206351:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575309.206353:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575309.206358:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575309.206362:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f9f8.
00000400:00000010:2.0:1788575309.206367:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575309.206370:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393199872, portal 10
00000100:00000001:2.0:1788575309.206372:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575309.206374:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:2.0:1788575309.206381:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575309.206389:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575309.206424:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575309.206428:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65908.
00000100:00000200:2.0:1788575309.206433:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393199872, offset 0
00000400:00000200:2.0:1788575309.206439:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575309.206445:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575309.206449:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575309.206453:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575309.206456:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575309.206459:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575309.206462:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575309.206465:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330817:330817:255:4294967295] 192.168.202.110@tcp LPNI seq info [330774:330774:7:4294967295]
00000400:00000200:2.0:1788575309.206471:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575309.206479:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575309.206484:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575309.206488:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133e00.
00000800:00000200:2.0:1788575309.206493:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575309.206498:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575309.206501:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575309.206506:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575309.206510:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575309.206513:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306912807
00000100:00000001:2.0:1788575309.206516:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306913721
00000100:00000001:2.0:1788575309.206519:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306912807
00000100:00000001:2.0:1788575309.206521:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306913721
00000100:00000001:2.0:1788575309.206524:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575309.206527:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575309.206529:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575309.206534:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575309.206542:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575309.206543:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575309.206546:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575309.206548:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575309.206550:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575309.206553:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200000:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575309.206557:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575309.206559:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575309.206561:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575309.206564:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575309.206568:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575309.206570:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575309.206574:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575309.206577:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575309.206581:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575309.206584:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393200000, portal 4
00000100:00000001:2.0:1788575309.206586:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575309.206587:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:2.0:1788575309.206593:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575309.206601:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575309.206603:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575309.206605:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65330.
00000100:00000200:2.0:1788575309.206608:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393200000, offset 0
00000400:00000200:2.0:1788575309.206613:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575309.206617:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575309.206622:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575309.206626:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575309.206629:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575309.206631:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575309.206634:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575309.206638:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330818:330818:254:4294967295] 192.168.202.110@tcp LPNI seq info [330775:330775:6:4294967295]
00000400:00000200:2.0:1788575309.206643:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575309.206650:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575309.206655:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575309.206659:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133700.
00000800:00000200:2.0:1788575309.206662:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575309.206666:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575309.206669:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575309.206672:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575309.206676:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575309.206678:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306912953
00000100:00000001:2.0:1788575309.206681:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306913721
00000100:00000001:2.0:1788575309.206683:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306912953
00000100:00000001:2.0:1788575309.206685:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306913721
00000100:00000001:2.0:1788575309.206688:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575309.206690:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575309.206693:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575309.206695:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575309.206699:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575309.206700:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575309.206703:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575309.206704:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575309.206706:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575309.206708:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575309.589513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575309.589520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575309.589524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575309.589525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575309.589528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575309.589529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575309.589600:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575309.589606:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575309.589609:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575309.717542:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575309.717551:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575309.717553:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575309.717558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575309.717561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575309.717566:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575309.717569:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575309.717571:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575309.717574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575310.165494:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306913721, at 4306913728
00000100:00000040:0.0:1788575310.165497:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306913721, at 4306913728
00000100:00000010:0.0:1788575310.165509:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000100:00000010:1.0:1788575310.165513:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00010000:00000001:3.0:1788575310.613505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575310.613511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575310.613517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575310.613519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575310.613522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575310.613524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575310.613588:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575310.613592:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575310.613596:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575310.741463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575310.741471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575310.741474:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575310.741479:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575310.741483:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575310.741486:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575310.741489:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575310.741492:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575310.741496:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575311.637458:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575311.637464:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575311.637468:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575311.637469:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575311.637471:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575311.637472:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575311.637524:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575311.637527:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575311.637530:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575311.765491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575311.765497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575311.765498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575311.765502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575311.765504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575311.765508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575311.765510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575311.765512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575311.765514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:3.0:1788575312.277686:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:3.0:1788575312.277727:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:3.0:1788575312.277736:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575312.277768:0:2352:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:3.0:1788575312.277774:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575312.277779:0:2352:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:3.0:1788575312.277784:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1788575312.277788:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133d00.
00000400:00000200:3.0:1788575312.277795:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1788575312.277801:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1788575312.277811:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65088
00000400:00000010:3.0:1788575312.277814:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65088.
00000100:00000001:3.0:1788575312.277821:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575312.277833:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575312.277882:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575312.277890:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575312.277917:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575312.277920:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:3.0:1788575312.277932:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1788575312.277935:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133e00.
00000400:00000200:3.0:1788575312.277939:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1788575312.277943:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1788575312.277947:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65908
00000400:00000010:3.0:1788575312.277949:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65908.
00000100:00000001:2.0:1788575312.277950:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575312.277952:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:2.0:1788575312.277955:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000200:3.0:1788575312.277958:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.277959:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575312.277967:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575309/real 1788575312]  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575312.277968:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575312.277972:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575312.277981:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575312.277982:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000400:00000001:2.0:1788575312.277983:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575312.277985:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000800:00000100:3.0:1788575312.277987:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000400:00000010:2.0:1788575312.277989:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000800:00000010:3.0:1788575312.277990:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133700.
00000400:00000200:2.0:1788575312.277993:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000200:3.0:1788575312.277994:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000010:2.0:1788575312.277997:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000400:00000200:3.0:1788575312.277998:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:2.0:1788575312.278000:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000400:00000200:3.0:1788575312.278001:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65330
00000400:00000010:3.0:1788575312.278004:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65330.
00000100:00000200:2.0:1788575312.278005:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575312.278007:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575312.278012:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575312.278016:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575312.278022:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1788575312.278026:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:3.0:1788575312.278026:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278028:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278031:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575312.278033:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575312.278034:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575312.278035:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000100:00100000:2.0:1788575312.278039:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278049:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575312.278055:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278066:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575312.278070:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575312.278074:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278078:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575312.278081:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575312.278087:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575312.278093:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575312.278096:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575312.278102:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575312.278112:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393199744:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575312.278118:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575312.278121:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575312.278124:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393199744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278132:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575312.278135:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575312.278141:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575312.278146:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575312.278149:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575312.278151:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575312.278154:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575312.278157:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575312.278162:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575312.278164:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575312.278166:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:2.0:1788575312.278170:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575312.278172:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278174:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575312.278177:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575312.278178:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575312.278180:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575312.278185:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575309/real 1788575312]  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575312.278195:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575312.278196:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575312.278198:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f9f8.
00000400:00000200:2.0:1788575312.278201:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575312.278203:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575312.278205:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575312.278209:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575312.278218:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278225:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575312.278227:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278230:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575312.278232:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575312.278237:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278244:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575312.278248:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278257:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575312.278259:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278261:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575312.278263:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278265:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575312.278267:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575312.278271:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575312.278275:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575312.278278:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575312.278283:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575312.278292:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393199872:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575312.278297:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575312.278301:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575309/real 1788575312]  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575312.278310:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575312.278312:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575312.278313:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575312.278316:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575312.278318:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575312.278321:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575312.278325:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575312.278336:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278344:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575312.278346:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278348:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575312.278350:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575312.278354:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278362:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575312.278367:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278374:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575312.278377:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278378:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575312.278380:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278382:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575312.278385:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575312.278388:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575312.278425:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575312.278429:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575312.278435:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575312.278445:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200000:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575312.278450:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278452:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575312.278456:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393199872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278464:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575312.278465:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575312.278468:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575312.278471:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575312.278473:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575312.278475:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575312.278477:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575312.278479:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575312.278482:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575312.278484:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575312.278485:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788575312.278488:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575312.278489:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278491:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575312.278496:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393200000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575334 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575312.278505:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575312.278506:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575312.278508:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575312.278511:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575312.278513:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575312.278515:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575312.278517:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575312.278519:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575312.278521:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575312.278523:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575312.278524:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:2.0:1788575312.278527:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575312.278528:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278530:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.278539:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575312.278541:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575312.278543:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575312.278544:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575312.661509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575312.661514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575312.661518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575312.661519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575312.661522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575312.661523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575312.661556:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575312.661559:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575312.661562:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575312.789485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575312.789490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575312.789492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575312.789497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575312.789499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575312.789502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.789504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575312.789506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575312.789508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575313.685507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575313.685514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575313.685520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575313.685521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575313.685526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575313.685527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575313.685572:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575313.685577:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575313.685581:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575313.813501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575313.813506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575313.813507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575313.813511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575313.813514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575313.813517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575313.813520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575313.813521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575313.813523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575314.325521:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575314.325531:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575314.325534:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575314.325538:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575314.325543:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575314.325550:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12245
00000400:00000200:1.0:1788575314.325559:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575314.325566:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575314.325570:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575314.325573:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.325577:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575314.325580:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.325583:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575314.325587:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575314.325589:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575314.325592:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575314.325595:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575314.325599:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575314.325601:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575314.325605:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.325608:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575314.325611:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.325613:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575314.325617:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575314.325621:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575314.325624:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575314.325627:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575314.325630:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575314.325633:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575314.325636:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575314.325639:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575314.325645:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788575314.325649:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575314.325653:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575314.325655:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575314.325658:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575314.325660:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575314.325869:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575314.325879:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788575314.325887:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575314.325898:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575314.325909:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575314.325914:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575314.325919:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575314.325946:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575314.325950:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575314.325954:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575314.325956:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575314.325958:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575314.325961:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575314.325965:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12245
00000400:00000200:1.0:1788575314.325969:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575314.325973:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575314.325975:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575314.325979:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.325981:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575314.325983:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.325985:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575314.325988:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575314.325989:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575314.325991:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575314.325993:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575314.325995:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575314.325997:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575314.326000:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.326001:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575314.326003:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.326005:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575314.326008:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575314.326010:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575314.326012:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575314.326014:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575314.326021:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575314.326023:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575314.326025:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575314.326031:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575314.326034:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575314.326036:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575314.326039:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788575314.326041:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575314.326044:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575314.326046:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575314.326047:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575314.326049:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575314.326051:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575314.326055:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788575314.326059:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575314.326063:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575314.326071:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575314.326073:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575314.326077:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575314.326084:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575314.326087:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575314.326090:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575314.326091:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575314.326093:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575314.326096:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575314.326098:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12245
00000400:00000200:1.0:1788575314.326102:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575314.326105:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575314.326107:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575314.326110:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.326112:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575314.326114:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.326117:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575314.326119:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575314.326120:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575314.326122:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575314.326124:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575314.326126:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575314.326130:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575314.326133:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.326135:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575314.326136:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575314.326139:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575314.326142:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575314.326146:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575314.326149:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575314.326151:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575314.326154:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575314.326157:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.326157:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788575314.326161:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575314.326164:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000008:00100000:1.0:1788575314.326165:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575314.326168:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000008:00000001:1.0:1788575314.326170:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575314.326174:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575314.326179:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1788575314.326180:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788575314.326186:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575314.326191:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575314.326195:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.326198:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575314.326199:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575314.326203:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575314.326203:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575314.326207:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575314.326210:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:1.0:1788575314.326214:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
02000000:00000001:2.0:1788575314.326216:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788575314.326220:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575314.326221:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575314.326230:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393200128:192.168.202.110@tcp:250:kworker.0
00000100:00100000:1.0:1788575314.326231:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575314.326238:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1788575314.326249:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575314.326254:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575314.326257:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:1.0:1788575314.326259:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575314.326263:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575314.326267:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575314.326275:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
00000100:00000001:1.0:1788575314.326276:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575314.326282:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575314.326283:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575314.326289:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575314.326294:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788575314.326297:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12255)
00000400:00000010:2.0:1788575314.326301:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575314.326310:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575314.326319:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393200128, portal 25
00000100:00000001:2.0:1788575314.326322:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575314.326324:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
00000100:00000040:2.0:1788575314.326333:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326342:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575314.326346:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575314.326349:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575314.326353:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393200128, offset 0
00000400:00000200:2.0:1788575314.326361:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575314.326384:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575314.326444:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575314.326450:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575314.326455:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575314.326460:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575314.326463:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575314.326467:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330819:330819:256:4294967295] 192.168.202.110@tcp LPNI seq info [330776:330776:8:4294967295]
00000400:00000200:2.0:1788575314.326472:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575314.326484:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575314.326491:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575314.326498:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575314.326507:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575314.326512:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575314.326517:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575314.326522:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575314.326527:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575314.326529:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575314.326536:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575314.326542:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575314.326550:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575314.326553:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575314.326557:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575314.326562:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326574:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575314.326578:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326587:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.326589:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575314.326596:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575314.326601:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575314.326604:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306918842
00000100:00000010:2.0:1788575314.326610:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c120.
00000100:00000001:2.0:1788575314.326617:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575314.326640:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306918842
00000100:00000010:2.0:1788575314.326646:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c780.
00000100:00000001:2.0:1788575314.326649:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575314.326653:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575314.326657:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.326659:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575314.326664:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575314/real 1788575314]  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575314.326690:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575314.326692:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575314.326695:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575314.326699:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575314.326700:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575314.326703:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575314.326708:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575314.326718:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326726:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575314.326729:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.326731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575314.326734:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575314.326738:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326746:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575314.326751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326774:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575314.326778:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.326780:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575314.326781:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.326784:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575314.326786:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575314.326789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575314.326794:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575314.326797:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575314.326802:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575314.326812:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393200128:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575314.326817:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575314.326821:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326828:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575314.326829:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575314.326831:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575314.326833:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575314.326835:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575314.326837:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200256:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575314.326840:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575314.326841:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575314.326842:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575314.326844:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575314.326847:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575314.326848:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575314.326851:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575314.326854:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575314.326857:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575314.326861:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393200256, portal 10
00000100:00000001:2.0:1788575314.326864:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575314.326865:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000040:2.0:1788575314.326872:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326879:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575314.326882:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575314.326884:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575314.326887:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393200256, offset 0
00000400:00000200:2.0:1788575314.326892:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575314.326896:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575314.326900:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575314.326903:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575314.326905:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575314.326908:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575314.326910:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575314.326914:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330820:330820:256:4294967295] 192.168.202.110@tcp LPNI seq info [330777:330777:8:4294967295]
00000400:00000200:2.0:1788575314.326918:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575314.326925:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575314.326930:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575314.326934:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575314.326937:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575314.326940:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575314.326945:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575314.326948:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575314.326952:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575314.326954:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575314.326957:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575314.326960:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575314.326963:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575314.326965:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575314.326968:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575314.326971:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326978:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575314.326981:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.326987:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.326989:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575314.326991:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575314.326995:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575314.326999:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575314.327000:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306917927
00000100:00000001:2.0:1788575314.327003:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306918842
00000100:00000001:2.0:1788575314.327005:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306917927
00000100:00000001:2.0:1788575314.327008:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306918842
00000100:00000001:2.0:1788575314.327010:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575314.327012:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.327014:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575314.327018:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575314/real 1788575314]  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575314.327032:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575314.327033:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575314.327035:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575314.327037:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575314.327039:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575314.327042:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575314.327045:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575314.327055:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327063:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575314.327064:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327066:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575314.327068:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575314.327071:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327078:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575314.327083:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327090:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575314.327093:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327095:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575314.327097:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327099:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575314.327101:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575314.327105:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575314.327109:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575314.327112:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575314.327117:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575314.327126:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200256:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575314.327132:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327135:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575314.327139:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393200128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327146:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575314.327149:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575314.327152:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575314.327156:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575314.327158:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575314.327159:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575314.327162:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575314.327165:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:2.0:1788575314.327169:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575314.327172:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575314.327174:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:2.0:1788575314.327178:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575314.327179:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327181:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575314.327184:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393200256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327192:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575314.327193:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575314.327195:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575314.327197:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575314.327199:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575314.327200:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575314.327202:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575314.327204:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:2.0:1788575314.327206:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575314.327207:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575314.327209:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:2.0:1788575314.327211:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575314.327212:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327215:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327223:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575314.327226:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.327228:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575314.327230:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575314.327232:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575314.327236:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327244:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575314.327246:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575314.327248:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575314.327250:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575314.327252:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575314.327256:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575314.327259:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575314.327261:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575314.327263:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575314.327271:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575314.327275:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575314.327277:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575314.327281:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575314.327284:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575314.327288:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575314.327292:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393200384, portal 4
00000100:00000001:2.0:1788575314.327294:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575314.327296:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:2.0:1788575314.327303:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327311:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575314.327313:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575314.327315:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575314.327319:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393200384, offset 0
00000400:00000200:2.0:1788575314.327323:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575314.327328:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575314.327332:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575314.327335:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575314.327338:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575314.327341:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575314.327343:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575314.327347:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330821:330821:256:4294967295] 192.168.202.110@tcp LPNI seq info [330778:330778:8:4294967295]
00000400:00000200:2.0:1788575314.327351:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575314.327358:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575314.327363:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575314.327367:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575314.327371:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575314.327374:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575314.327378:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575314.327381:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575314.327385:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575314.327387:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575314.327429:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575314.327435:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575314.327438:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575314.327440:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575314.327444:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575314.327448:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327457:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575314.327461:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327468:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.327470:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575314.327474:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575314.327478:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575314.327482:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575314.327484:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306918074
00000100:00000001:2.0:1788575314.327487:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306918842
00000100:00000001:2.0:1788575314.327489:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306918074
00000100:00000001:2.0:1788575314.327491:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306918842
00000100:00000001:2.0:1788575314.327493:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575314.327495:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.327497:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575314.327502:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575314/real 1788575314]  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575314.327511:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575314.327513:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575314.327514:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575314.327517:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575314.327519:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575314.327521:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575314.327525:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575314.327534:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327540:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575314.327542:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327544:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575314.327545:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575314.327549:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327556:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575314.327560:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327568:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575314.327571:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327573:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575314.327574:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327576:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575314.327578:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575314.327581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575314.327585:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575314.327587:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575314.327591:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575314.327599:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575314.327603:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327611:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575314.327615:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393200384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575314.327622:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575314.327623:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575314.327626:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575314.327629:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575314.327630:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575314.327632:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575314.327633:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575314.327634:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:2.0:1788575314.327636:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575314.327637:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575314.327638:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:2.0:1788575314.327640:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575314.327641:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327643:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.327646:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575314.327647:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575314.327648:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575314.327649:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575314.709537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575314.709543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575314.709552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575314.709557:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575314.709559:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575314.709562:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575314.709565:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575314.709567:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575314.709568:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575314.709570:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575314.709573:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575314.709574:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575314.709575:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575314.709577:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575314.709578:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575314.709579:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575314.709581:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575314.709582:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575314.709704:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575314.709711:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575314.709716:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575314.837543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575314.837552:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575314.837555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575314.837561:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575314.837565:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575314.837570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.837573:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575314.837575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575314.837579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575315.285508:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306918842, at 4306918848
00000100:00000040:1.0:1788575315.285508:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306918842, at 4306918848
00000100:00000010:0.0:1788575315.285533:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00000100:00000010:1.0:1788575315.285537:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00010000:00000001:3.0:1788575315.733505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575315.733512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575315.733517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575315.733519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575315.733523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575315.733524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575315.733579:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575315.733583:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575315.733586:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575315.861557:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575315.861568:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575315.861571:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575315.861579:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575315.861584:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575315.861588:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575315.861592:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575315.861595:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575315.861599:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575316.757488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575316.757495:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575316.757497:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575316.757499:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575316.757503:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575316.757505:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575316.757507:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575316.757508:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575316.757511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575316.757512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575316.757514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575316.757516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575316.757578:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575316.757581:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575316.757584:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575316.885522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575316.885529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575316.885531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575316.885535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575316.885538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575316.885541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575316.885543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575316.885545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575316.885547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575317.781509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575317.781515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575317.781518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575317.781519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575317.781522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575317.781523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575317.781610:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575317.781614:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575317.781618:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575317.909542:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575317.909548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575317.909550:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575317.909555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575317.909559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575317.909594:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575317.909598:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575317.909600:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575317.909604:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575318.805508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575318.805514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575318.805520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575318.805521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575318.805525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575318.805527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575318.805606:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575318.805609:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575318.805612:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575318.933489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575318.933497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575318.933500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575318.933505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575318.933509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575318.933513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575318.933515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575318.933517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575318.933519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575319.445576:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575319.445586:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575319.445589:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575319.445593:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575319.445598:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575319.445605:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12250
00000400:00000200:2.0:1788575319.445615:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575319.445622:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575319.445625:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575319.445628:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575319.445631:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575319.445634:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575319.445637:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575319.445641:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575319.445643:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575319.445647:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575319.445650:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575319.445652:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575319.445655:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575319.445658:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575319.445660:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575319.445662:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575319.445664:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575319.445667:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575319.445671:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575319.445673:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575319.445676:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575319.445679:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575319.445682:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575319.445684:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575319.445686:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575319.445695:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01f80.
00000020:00000040:2.0:1788575319.445699:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575319.445703:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575319.445705:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575319.445708:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575319.445710:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575319.445715:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575319.445722:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000001:2.0:1788575319.445728:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575319.445736:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575319.445746:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575319.445750:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575319.445754:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8c8ec7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575319.445786:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575319.445790:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575319.445793:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575319.445794:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575319.445796:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575319.445799:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575319.445801:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12250
00000400:00000200:2.0:1788575319.445805:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575319.445808:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575319.445811:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575319.445815:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575319.445817:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575319.445820:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575319.445822:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575319.445824:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575319.445826:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575319.445828:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575319.445830:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575319.445832:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575319.445834:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575319.445835:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575319.445838:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1788575319.445842:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575319.445842:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788575319.445847:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575319.445852:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575319.445853:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575319.445856:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575319.445862:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575319.445869:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575319.445874:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575319.445878:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575319.445878:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575319.445884:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575319.445887:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575319.445889:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575319.445891:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575319.445894:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575319.445895:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575319.445901:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1788575319.445902:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575319.445908:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575319.445910:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393200512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575319.445912:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575319.445916:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03800.
00000020:00000040:2.0:1788575319.445919:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575319.445922:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575319.445925:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575319.445929:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1788575319.445931:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575319.445932:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575319.445937:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575319.445939:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:2.0:1788575319.445941:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575319.445943:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575319.445947:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575319.445949:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:2.0:1788575319.445956:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:3.0:1788575319.445958:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575319.445963:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575319.445971:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575319.445973:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575319.445977:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575319.445978:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000040:2.0:1788575319.445985:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014d98894] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788575319.445988:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575319.445997:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393200512, portal 25
00000100:00000001:2.0:1788575319.446000:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575319.446002:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1788575319.446005:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575319.446009:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575319.446011:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575319.446014:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575319.446015:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000001:2.0:1788575319.446019:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575319.446027:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12250
00000100:00000040:3.0:1788575319.446028:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575319.446036:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575319.446044:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1788575319.446048:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1788575319.446049:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575319.446056:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575319.446057:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575319.446062:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1788575319.446063:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000040:2.0:1788575319.446066:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575319.446070:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:3.0:1788575319.446070:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393200512, offset 0
00000020:00000001:2.0:1788575319.446077:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575319.446079:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575319.446080:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575319.446082:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575319.446085:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:2.0:1788575319.446086:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575319.446091:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575319.446095:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575319.446099:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575319.446099:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575319.446105:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575319.446108:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1788575319.446110:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575319.446114:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:2.0:1788575319.446116:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575319.446121:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1788575319.446124:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575319.446129:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1788575319.446130:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575319.446133:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575319.446136:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575319.446138:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575319.446142:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575319.446145:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000008:00100000:2.0:1788575319.446146:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575319.446151:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575319.446155:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575319.446156:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330822:330822:256:4294967295] 192.168.202.110@tcp LPNI seq info [330779:330779:8:4294967295]
00000100:00000010:2.0:1788575319.446160:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02a00.
00000400:00000200:3.0:1788575319.446165:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575319.446167:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575319.446172:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575319.446173:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575319.446176:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575319.446179:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575319.446182:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1788575319.446183:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575319.446190:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:2.0:1788575319.446194:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575319.446199:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00100000:2.0:1788575319.446200:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575319.446208:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000010:3.0:1788575319.446208:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00000001:2.0:1788575319.446212:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575319.446216:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cce910e4] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788575319.446222:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575319.446225:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575319.446229:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1788575319.446230:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575319.446239:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:3.1:1788575319.446241:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575319.446246:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12260)
00000800:00000200:3.1:1788575319.446247:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575319.446255:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575319.446258:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575319.446267:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575319.446273:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575319.446279:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575319.446282:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575319.446286:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575319.446291:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446302:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575319.446306:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446312:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575319.446313:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575319.446320:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575319.446324:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575319.446326:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306923961
00000100:00000010:3.0:1788575319.446331:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788575319.446336:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575319.446347:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306923961
00000100:00000010:3.0:1788575319.446355:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575319.446358:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575319.446360:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575319.446363:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575319.446364:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575319.446369:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575319/real 1788575319]  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575319.446377:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575319.446379:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575319.446382:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575319.446384:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575319.446385:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575319.446388:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575319.446418:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575319.446425:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446430:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575319.446432:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.446434:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575319.446436:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575319.446439:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446450:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575319.446467:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446484:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575319.446488:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.446490:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575319.446491:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.446493:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575319.446495:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575319.446499:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575319.446503:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575319.446506:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575319.446521:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575319.446531:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393200512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575319.446537:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.446539:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575319.446543:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393200512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446550:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575319.446564:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575319.446567:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575319.446571:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575319.446573:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575319.446575:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575319.446578:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575319.446580:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575319.446582:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575319.446584:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575319.446585:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575319.446588:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575319.446589:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.446590:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.446596:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575319.446598:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575319.446599:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575319.446601:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575319.446602:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575319.446605:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446610:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575319.446611:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575319.446613:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575319.446614:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575319.446616:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575319.446618:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575319.446621:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575319.446622:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575319.446623:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575319.446625:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575319.446628:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575319.446630:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575319.446634:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575319.446637:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575319.446641:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575319.446644:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393200640, portal 10
00000100:00000001:3.0:1788575319.446646:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575319.446647:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:3.0:1788575319.446654:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446661:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575319.446663:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575319.446666:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575319.446668:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393200640, offset 0
00000400:00000200:3.0:1788575319.446673:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575319.446677:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575319.446681:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575319.446684:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575319.446687:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575319.446689:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575319.446692:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575319.446695:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330823:330823:256:4294967295] 192.168.202.110@tcp LPNI seq info [330780:330780:8:4294967295]
00000400:00000200:3.0:1788575319.446699:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575319.446707:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575319.446712:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575319.446716:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575319.446720:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575319.446722:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575319.446725:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575319.446727:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575319.446730:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575319.446731:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575319.446734:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575319.446736:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575319.446739:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575319.446740:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575319.446742:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575319.446744:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446750:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575319.446752:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446757:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575319.446758:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575319.446761:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575319.446763:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575319.446766:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575319.446768:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306923047
00000100:00000001:3.0:1788575319.446771:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306923961
00000100:00000001:3.0:1788575319.446773:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306923047
00000100:00000001:3.0:1788575319.446775:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306923961
00000100:00000001:3.0:1788575319.446778:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575319.446780:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575319.446782:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575319.446786:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575319/real 1788575319]  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575319.446802:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575319.446803:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575319.446804:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575319.446806:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575319.446807:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575319.446809:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575319.446811:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575319.446817:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446821:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575319.446822:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.446823:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575319.446824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575319.446826:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446831:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575319.446833:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446838:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575319.446840:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.446841:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575319.446841:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.446843:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575319.446844:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575319.446846:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575319.446849:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575319.446851:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575319.446856:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575319.446864:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575319.446868:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575319.446872:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446879:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575319.446881:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575319.446883:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575319.446885:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575319.446887:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575319.446891:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200768:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575319.446895:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575319.446897:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575319.446898:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575319.446900:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575319.446904:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575319.446907:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575319.446911:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575319.446914:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575319.446917:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575319.446921:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393200768, portal 4
00000100:00000001:3.0:1788575319.446923:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575319.446924:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:3.0:1788575319.446931:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.446938:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575319.446940:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575319.446942:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575319.446945:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393200768, offset 0
00000400:00000200:3.0:1788575319.446949:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575319.446953:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575319.446957:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575319.446960:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575319.446963:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575319.446965:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575319.446968:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575319.446971:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330824:330824:256:4294967295] 192.168.202.110@tcp LPNI seq info [330781:330781:8:4294967295]
00000400:00000200:3.0:1788575319.446976:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575319.446981:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575319.446984:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575319.446986:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575319.446989:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575319.446991:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575319.446994:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575319.446996:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575319.446999:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575319.447000:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575319.447002:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575319.447004:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575319.447007:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575319.447008:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575319.447009:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575319.447012:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.447017:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575319.447020:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.447024:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575319.447026:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575319.447029:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575319.447032:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575319.447036:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575319.447038:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306923193
00000100:00000001:3.0:1788575319.447041:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306923961
00000100:00000001:3.0:1788575319.447043:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306923193
00000100:00000001:3.0:1788575319.447045:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306923961
00000100:00000001:3.0:1788575319.447047:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575319.447049:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575319.447051:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575319.447056:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575319/real 1788575319]  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575319.447065:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575319.447066:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575319.447068:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575319.447071:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575319.447073:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575319.447074:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575319.447078:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575319.447086:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.447093:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575319.447095:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.447097:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575319.447098:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575319.447102:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.447110:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575319.447114:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.447122:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575319.447125:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.447127:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575319.447129:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.447131:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575319.447132:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575319.447136:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575319.447139:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575319.447142:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575319.447146:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575319.447154:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393200768:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575319.447158:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.447160:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575319.447164:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393200640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.447171:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575319.447173:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575319.447175:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575319.447178:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575319.447180:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575319.447181:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575319.447183:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575319.447185:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575319.447188:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575319.447189:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575319.447191:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:3.0:1788575319.447194:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575319.447196:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.447198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575319.447201:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393200768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575319.447214:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575319.447215:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575319.447217:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575319.447220:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575319.447221:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575319.447223:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575319.447225:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575319.447226:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575319.447228:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575319.447230:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575319.447231:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:3.0:1788575319.447234:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575319.447235:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.447237:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575319.447241:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575319.447243:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575319.447245:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575319.447247:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575319.829468:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575319.829475:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575319.829481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575319.829483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575319.829487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575319.829488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575319.829541:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575319.829544:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575319.829547:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575319.957519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575319.957525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575319.957527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575319.957531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575319.957535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575319.957538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575319.957540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575319.957543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575319.957545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575320.405527:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306923961, at 4306923968
00000100:00000040:0.0:1788575320.405533:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306923961, at 4306923968
00000100:00000010:1.0:1788575320.405548:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000010:0.0:1788575320.405553:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00010000:00000001:3.0:1788575320.853518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575320.853525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575320.853532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575320.853534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575320.853538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575320.853540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575320.853590:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575320.853595:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575320.853598:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575320.981456:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575320.981473:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575320.981475:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575320.981479:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575320.981481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575320.981484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575320.981486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575320.981489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575320.981491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575321.877516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575321.877521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575321.877524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575321.877525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575321.877527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575321.877528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575321.877591:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575321.877595:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575321.877599:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575322.005503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575322.005508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575322.005510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575322.005515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575322.005518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575322.005521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575322.005524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575322.005526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575322.005528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575322.901529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575322.901534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575322.901538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575322.901539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575322.901541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575322.901542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575322.901590:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575322.901592:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575322.901595:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575323.029517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575323.029523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575323.029524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575323.029529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575323.029532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575323.029537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575323.029544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575323.029547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575323.029550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575323.925464:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575323.925470:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575323.925475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575323.925477:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575323.925479:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575323.925482:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575323.925484:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575323.925489:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575323.925491:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575323.925492:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575323.925495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575323.925496:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575323.925497:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575323.925498:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575323.925499:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575323.925500:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575323.925501:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575323.925503:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575323.925542:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575323.925545:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575323.925547:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575324.053501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575324.053508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575324.053510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575324.053514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575324.053516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575324.053520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575324.053523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575324.053526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575324.053529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575324.565512:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575324.565523:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575324.565527:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575324.565531:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575324.565536:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575324.565544:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12255
00000400:00000200:2.0:1788575324.565554:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575324.565561:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575324.565565:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575324.565568:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.565572:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575324.565575:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.565578:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575324.565583:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575324.565585:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575324.565588:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575324.565591:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575324.565594:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575324.565597:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575324.565601:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.565603:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575324.565605:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.565607:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575324.565611:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575324.565615:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575324.565618:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575324.565621:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575324.565625:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575324.565628:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575324.565631:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575324.565633:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575324.565643:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02680.
00000020:00000040:2.0:1788575324.565647:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575324.565652:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575324.565654:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575324.565657:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575324.565660:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575324.565664:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575324.565671:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000100:00000001:2.0:1788575324.565678:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575324.565688:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575324.565699:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575324.565703:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575324.565708:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000450d1599] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575324.565731:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575324.565735:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575324.565739:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575324.565740:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575324.565742:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575324.565745:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575324.565748:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12255
00000400:00000200:2.0:1788575324.565753:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575324.565757:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575324.565760:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575324.565764:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.565767:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575324.565771:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.565773:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575324.565775:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575324.565776:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575324.565780:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575324.565781:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1788575324.565783:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575324.565786:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1788575324.565788:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575324.565793:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575324.565796:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575324.565798:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575324.565808:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.565816:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575324.565816:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575324.565820:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575324.565822:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.565827:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575324.565827:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575324.565831:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1788575324.565833:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575324.565837:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575324.565840:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:3.0:1788575324.565845:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393200896:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575324.565847:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575324.565851:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575324.565855:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575324.565859:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575324.565862:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575324.565864:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575324.565866:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575324.565868:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575324.565871:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:2.0:1788575324.565875:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:3.0:1788575324.565879:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
00000008:00000001:2.0:1788575324.565880:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575324.565884:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575324.565886:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575324.565892:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00700.
00000400:00000200:3.0:1788575324.565894:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575324.565898:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:3.0:1788575324.565902:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
02000000:00000001:2.0:1788575324.565904:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575324.565907:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575324.565910:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575324.565912:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1788575324.565912:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
02000000:00000001:2.0:1788575324.565915:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575324.565921:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393200896, portal 25
02000000:00000010:2.0:1788575324.565923:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12c00.
00000100:00000001:3.0:1788575324.565926:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575324.565930:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.565930:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:3.0:1788575324.565941:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575324.565943:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575324.565972:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1788575324.565973:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575324.565978:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575324.565979:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575324.565985:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000040:2.0:1788575324.565989:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e69941e] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575324.565991:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393200896, offset 0
00000400:00000200:3.0:1788575324.566006:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575324.566017:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575324.566018:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575324.566023:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575324.566030:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:3.0:1788575324.566030:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575324.566038:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575324.566042:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1788575324.566044:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575324.566049:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575324.566049:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575324.566055:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.566056:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575324.566063:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12255
00000400:00000200:3.0:1788575324.566066:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330825:330825:256:4294967295] 192.168.202.110@tcp LPNI seq info [330782:330782:8:4294967295]
00000400:00000200:3.0:1788575324.566073:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575324.566079:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575324.566089:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1788575324.566090:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1788575324.566094:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575324.566101:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.566106:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575324.566106:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575324.566112:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:3.0:1788575324.566115:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00000001:2.0:1788575324.566116:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575324.566121:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575324.566123:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575324.566127:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.0:1788575324.566128:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1788575324.566130:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:3.0:1788575324.566134:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1788575324.566135:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575324.566141:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.1:1788575324.566147:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575324.566149:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575324.566153:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1788575324.566154:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575324.566160:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575324.566164:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000100:3.0:1788575324.566165:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:2.0:1788575324.566169:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000010:3.0:1788575324.566171:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000100:00080000:2.0:1788575324.566176:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575324.566180:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575324.566180:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1788575324.566186:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575324.566191:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1788575324.566192:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575324.566197:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575324.566198:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000100:00000001:2.0:1788575324.566199:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1788575324.566203:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000008:00100000:2.0:1788575324.566204:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575324.566210:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.566211:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:2.0:1788575324.566212:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575324.566218:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1788575324.566220:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000020:00000040:2.0:1788575324.566226:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575324.566232:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575324.566236:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.566238:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1788575324.566239:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575324.566242:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1788575324.566246:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575324.566247:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575324.566253:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788575324.566259:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.566261:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.566265:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00100000:2.0:1788575324.566268:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575324.566273:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575324.566281:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575324.566282:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575324.566286:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.566288:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306929081
00000100:00000040:2.0:1788575324.566294:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1788575324.566298:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575324.566307:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575324.566309:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575324.566314:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575324.566318:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575324.566323:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12265)
00000100:00000001:3.0:1788575324.566326:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306929081
00000100:00000010:3.0:1788575324.566332:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575324.566336:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575324.566341:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575324.566344:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.566364:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575324.566375:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575324/real 1788575324]  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575324.566387:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575324.566389:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575324.566416:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575324.566420:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575324.566422:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575324.566425:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575324.566430:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575324.566439:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566446:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575324.566448:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.566452:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575324.566454:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575324.566459:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566466:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575324.566471:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566479:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575324.566482:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.566484:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575324.566486:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.566488:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575324.566490:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575324.566493:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575324.566497:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575324.566499:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575324.566504:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575324.566513:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393200896:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575324.566518:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.566520:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575324.566524:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393200896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566538:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575324.566541:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575324.566543:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575324.566546:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575324.566549:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575324.566550:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575324.566553:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575324.566555:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575324.566559:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575324.566561:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575324.566563:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575324.566566:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575324.566568:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.566570:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.566576:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575324.566579:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.566581:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575324.566582:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575324.566584:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575324.566588:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566594:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575324.566596:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575324.566598:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575324.566600:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575324.566603:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575324.566606:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201024:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575324.566609:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575324.566611:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575324.566613:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575324.566615:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575324.566618:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575324.566621:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575324.566624:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575324.566628:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575324.566631:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575324.566634:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393201024, portal 10
00000100:00000001:3.0:1788575324.566637:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575324.566638:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:3.0:1788575324.566645:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566652:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575324.566654:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575324.566656:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575324.566659:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393201024, offset 0
00000400:00000200:3.0:1788575324.566663:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575324.566667:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575324.566672:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575324.566675:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575324.566678:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575324.566680:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575324.566683:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575324.566686:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330826:330826:256:4294967295] 192.168.202.110@tcp LPNI seq info [330783:330783:8:4294967295]
00000400:00000200:3.0:1788575324.566691:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575324.566698:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575324.566703:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575324.566706:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575324.566710:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575324.566713:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788575324.566717:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575324.566720:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1788575324.566724:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575324.566726:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575324.566729:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575324.566733:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575324.566736:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575324.566738:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575324.566741:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575324.566745:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566753:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575324.566757:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566763:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.566765:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575324.566769:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575324.566772:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575324.566776:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575324.566778:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306928167
00000100:00000001:3.0:1788575324.566781:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306929081
00000100:00000001:3.0:1788575324.566783:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306928167
00000100:00000001:3.0:1788575324.566786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306929081
00000100:00000001:3.0:1788575324.566788:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575324.566790:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.566793:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575324.566796:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575324/real 1788575324]  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575324.566805:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575324.566806:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575324.566808:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575324.566811:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575324.566813:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575324.566815:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575324.566819:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575324.566827:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566839:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575324.566841:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.566842:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575324.566844:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575324.566847:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566854:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575324.566858:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566865:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575324.566867:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.566869:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575324.566871:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.566873:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575324.566875:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575324.566877:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575324.566881:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575324.566883:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575324.566887:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575324.566895:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201024:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575324.566899:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575324.566902:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566909:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575324.566910:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575324.566913:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575324.566915:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575324.566918:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575324.566921:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201152:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575324.566924:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575324.566926:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575324.566927:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575324.566929:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575324.566933:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575324.566935:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575324.566938:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575324.566941:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575324.566945:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575324.566948:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393201152, portal 4
00000100:00000001:3.0:1788575324.566950:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575324.566952:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575324.566957:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.566964:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575324.566967:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575324.566969:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575324.566971:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393201152, offset 0
00000400:00000200:3.0:1788575324.566975:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575324.566979:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575324.566982:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575324.566985:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575324.566988:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575324.566990:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575324.566993:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575324.566996:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330827:330827:256:4294967295] 192.168.202.110@tcp LPNI seq info [330784:330784:8:4294967295]
00000400:00000200:3.0:1788575324.567000:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575324.567007:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575324.567012:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575324.567015:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575324.567019:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575324.567022:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788575324.567026:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575324.567029:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1788575324.567033:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575324.567034:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575324.567037:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575324.567041:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575324.567043:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575324.567045:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575324.567048:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575324.567051:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.567059:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575324.567062:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.567069:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.567071:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575324.567074:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575324.567077:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575324.567080:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575324.567083:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306928313
00000100:00000001:3.0:1788575324.567085:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306929081
00000100:00000001:3.0:1788575324.567088:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306928313
00000100:00000001:3.0:1788575324.567090:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306929081
00000100:00000001:3.0:1788575324.567092:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575324.567094:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.567096:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575324.567100:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575324/real 1788575324]  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575324.567108:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575324.567109:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575324.567111:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575324.567114:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575324.567116:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575324.567118:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575324.567120:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575324.567129:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.567136:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575324.567137:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.567139:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575324.567141:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575324.567144:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.567151:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575324.567155:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.567162:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575324.567165:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.567166:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575324.567168:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.567170:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575324.567172:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575324.567174:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575324.567178:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575324.567180:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575324.567185:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575324.567193:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201152:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575324.567196:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.567198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575324.567202:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393201024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.567208:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575324.567210:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575324.567212:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575324.567214:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575324.567216:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575324.567217:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575324.567219:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575324.567221:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575324.567224:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575324.567226:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575324.567227:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575324.567230:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575324.567232:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.567233:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575324.567237:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393201152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575324.567243:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575324.567244:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575324.567246:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575324.567248:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575324.567249:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575324.567251:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575324.567253:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575324.567254:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575324.567256:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575324.567258:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575324.567259:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575324.567261:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575324.567263:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.567265:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575324.567269:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575324.567271:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575324.567273:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575324.567275:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575324.949514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575324.949521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575324.949527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575324.949528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575324.949532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575324.949533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575324.949591:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575324.949594:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575324.949597:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575325.077460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575325.077465:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575325.077467:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575325.077471:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575325.077474:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575325.077477:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575325.077479:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575325.077481:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575325.077483:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575325.525465:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306929081, at 4306929088
00000100:00000010:1.0:1788575325.525482:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000040:0.0:1788575325.525493:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306929081, at 4306929088
00000100:00000010:0.0:1788575325.525502:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00010000:00000001:3.0:1788575325.973484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575325.973488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575325.973492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575325.973493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575325.973495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575325.973496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575325.973549:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575325.973552:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575325.973555:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575326.101476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575326.101483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575326.101484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575326.101489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575326.101492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575326.101495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575326.101497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575326.101499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575326.101501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575326.997474:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575326.997481:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575326.997483:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575326.997486:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575326.997490:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575326.997492:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575326.997493:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575326.997495:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575326.997498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575326.997499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575326.997501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575326.997502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575326.997561:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575326.997565:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575326.997568:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575327.125517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575327.125525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575327.125528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575327.125533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575327.125537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575327.125541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575327.125544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575327.125546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575327.125549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788575327.637510:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575327.637517:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788575327.637521:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575327.637662:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575327.637672:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575327.637677:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575327.637683:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575327.637686:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575328.021517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575328.021524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575328.021529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575328.021530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575328.021534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575328.021536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575328.021617:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575328.021624:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575328.021627:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575328.149529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575328.149538:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575328.149539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575328.149545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575328.149549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575328.149553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575328.149556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575328.149558:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575328.149561:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575329.045517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575329.045525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575329.045531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575329.045532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575329.045535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575329.045538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575329.045595:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575329.045599:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575329.045604:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575329.174814:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575329.174821:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575329.174824:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575329.174830:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575329.174834:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575329.174837:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575329.174841:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575329.174844:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575329.174847:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575329.685550:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575329.685561:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575329.685565:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575329.685570:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575329.685576:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575329.685586:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12260
00000400:00000200:1.0:1788575329.685598:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575329.685606:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575329.685610:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575329.685614:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575329.685619:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575329.685622:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575329.685626:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575329.685631:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575329.685633:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575329.685637:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575329.685640:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575329.685643:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575329.685653:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575329.685656:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575329.685658:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575329.685659:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575329.685661:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575329.685663:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575329.685666:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575329.685669:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575329.685671:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575329.685675:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575329.685677:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575329.685679:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575329.685681:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575329.685692:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788575329.685696:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575329.685700:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575329.685701:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575329.685704:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575329.685707:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575329.685713:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575329.685720:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788575329.685727:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575329.685737:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575329.685746:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575329.685750:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575329.685754:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575329.685773:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575329.685776:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575329.685779:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575329.685780:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575329.685781:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575329.685784:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575329.685786:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12260
00000400:00000200:1.0:1788575329.685789:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575329.685791:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575329.685793:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575329.685795:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575329.685797:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575329.685799:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575329.685801:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575329.685803:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575329.685804:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575329.685812:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575329.685815:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575329.685817:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575329.685819:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575329.685822:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575329.685822:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788575329.685825:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575329.685828:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1788575329.685829:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575329.685833:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1788575329.685834:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575329.685839:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:3.0:1788575329.685843:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788575329.685846:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575329.685854:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575329.685861:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575329.685861:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575329.685864:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575329.685868:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1788575329.685869:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575329.685871:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575329.685877:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575329.685877:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575329.685884:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:3.0:1788575329.685885:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393201280:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575329.685891:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:1.0:1788575329.685892:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575329.685895:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:1.0:1788575329.685899:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575329.685899:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575329.685905:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788575329.685907:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575329.685913:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
02000000:00000010:3.0:1788575329.685916:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
00000020:00000040:1.0:1788575329.685917:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575329.685922:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575329.685923:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575329.685926:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575329.685928:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575329.685929:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575329.685931:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575329.685932:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575329.685936:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
02000000:00000010:1.0:1788575329.685939:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788575329.685943:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575329.685945:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00100000:1.0:1788575329.685951:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575329.685955:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393201280, portal 25
00000100:00000001:1.0:1788575329.685961:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575329.685961:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575329.685965:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00000001:1.0:1788575329.685966:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575329.685974:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575329.685976:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686006:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575329.686012:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575329.686017:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000001:1.0:1788575329.686019:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575329.686022:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393201280, offset 0
00000400:00000200:3.0:1788575329.686029:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788575329.686031:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575329.686035:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575329.686036:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575329.686037:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575329.686040:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575329.686042:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575329.686047:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12260
00000400:00000200:3.0:1788575329.686051:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788575329.686052:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575329.686057:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1788575329.686058:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1788575329.686061:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575329.686065:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:1.0:1788575329.686067:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575329.686070:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575329.686072:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:1.0:1788575329.686076:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575329.686079:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:1.0:1788575329.686084:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575329.686084:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330828:330828:256:4294967295] 192.168.202.110@tcp LPNI seq info [330785:330785:8:4294967295]
00000020:00000001:1.0:1788575329.686091:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1788575329.686092:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:1.0:1788575329.686093:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575329.686096:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575329.686098:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575329.686100:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575329.686104:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575329.686116:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575329.686120:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575329.686120:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1788575329.686130:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575329.686131:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575329.686134:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575329.686138:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000010:3.0:1788575329.686139:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00080000:1.0:1788575329.686144:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575329.686149:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575329.686150:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1788575329.686156:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.0:1788575329.686158:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
02000000:00000001:1.0:1788575329.686163:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.1:1788575329.686164:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1788575329.686171:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575329.686171:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575329.686179:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575329.686181:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000100:00000001:1.0:1788575329.686196:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575329.686201:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00100000:1.0:1788575329.686202:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575329.686207:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575329.686210:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575329.686214:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000010:1.0:1788575329.686216:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788575329.686220:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575329.686220:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
02000000:00000001:1.0:1788575329.686223:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:3.0:1788575329.686225:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
02000000:00000001:1.0:1788575329.686231:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.686231:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:1.0:1788575329.686235:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575329.686238:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:3.0:1788575329.686238:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788575329.686242:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575329.686251:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:3.0:1788575329.686253:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1788575329.686257:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575329.686260:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788575329.686267:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575329.686277:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.686283:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788575329.686286:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575329.686291:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.686296:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575329.686297:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575329.686309:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575329.686311:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575329.686315:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575329.686317:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306934201
00000100:00000040:1.0:1788575329.686320:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575329.686327:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12270)
00000100:00000010:3.0:1788575329.686327:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575329.686340:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575329.686348:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306934201
00000100:00000010:3.0:1788575329.686352:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575329.686358:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575329.686363:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575329.686366:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.686369:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575329.686375:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575329/real 1788575329]  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575329.686387:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575329.686389:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575329.686427:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575329.686430:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575329.686431:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575329.686434:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575329.686437:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575329.686444:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686449:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575329.686450:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686452:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575329.686453:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575329.686456:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686462:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575329.686465:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686471:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575329.686474:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686475:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575329.686476:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686478:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575329.686480:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575329.686482:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575329.686485:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575329.686486:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575329.686489:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575329.686494:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393201280:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575329.686498:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686500:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575329.686502:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393201280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686507:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575329.686510:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575329.686512:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575329.686515:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575329.686516:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575329.686518:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575329.686519:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575329.686522:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:3.0:1788575329.686525:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575329.686527:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575329.686528:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:3.0:1788575329.686531:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575329.686532:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686534:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686541:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575329.686543:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.686544:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575329.686545:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575329.686547:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575329.686549:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686554:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575329.686555:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575329.686557:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575329.686558:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575329.686560:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575329.686561:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575329.686564:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575329.686566:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575329.686567:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575329.686568:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575329.686571:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575329.686572:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575329.686575:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575329.686577:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575329.686580:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575329.686582:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393201408, portal 10
00000100:00000001:3.0:1788575329.686584:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575329.686585:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:3.0:1788575329.686590:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686595:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575329.686597:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575329.686598:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575329.686600:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393201408, offset 0
00000400:00000200:3.0:1788575329.686603:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575329.686606:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575329.686609:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575329.686612:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575329.686614:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575329.686616:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575329.686617:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575329.686620:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330829:330829:256:4294967295] 192.168.202.110@tcp LPNI seq info [330786:330786:8:4294967295]
00000400:00000200:3.0:1788575329.686623:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575329.686628:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575329.686631:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575329.686634:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575329.686637:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575329.686639:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575329.686642:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575329.686644:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575329.686646:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575329.686653:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575329.686656:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575329.686658:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575329.686661:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575329.686662:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575329.686664:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575329.686666:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686672:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575329.686674:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686679:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.686680:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575329.686682:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575329.686685:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575329.686688:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575329.686689:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306933287
00000100:00000001:3.0:1788575329.686692:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306934201
00000100:00000001:3.0:1788575329.686694:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306933287
00000100:00000001:3.0:1788575329.686695:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306934201
00000100:00000001:3.0:1788575329.686697:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575329.686698:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.686700:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575329.686703:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575329/real 1788575329]  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575329.686711:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575329.686712:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575329.686714:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575329.686716:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575329.686718:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575329.686720:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575329.686724:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575329.686733:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686740:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575329.686742:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686744:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575329.686746:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575329.686750:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686758:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575329.686762:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686770:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575329.686773:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686774:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575329.686776:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686777:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575329.686779:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575329.686781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575329.686783:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575329.686784:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575329.686787:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575329.686792:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575329.686795:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575329.686797:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686801:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575329.686802:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575329.686804:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575329.686805:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575329.686807:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575329.686809:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575329.686811:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575329.686812:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575329.686813:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575329.686815:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575329.686817:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575329.686819:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575329.686822:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575329.686823:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575329.686826:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575329.686828:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393201536, portal 4
00000100:00000001:3.0:1788575329.686830:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575329.686831:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:3.0:1788575329.686836:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686840:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575329.686842:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575329.686843:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575329.686845:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393201536, offset 0
00000400:00000200:3.0:1788575329.686848:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575329.686851:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575329.686853:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575329.686855:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575329.686857:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575329.686859:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575329.686860:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575329.686863:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330830:330830:256:4294967295] 192.168.202.110@tcp LPNI seq info [330787:330787:8:4294967295]
00000400:00000200:3.0:1788575329.686866:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575329.686870:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575329.686873:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575329.686875:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575329.686878:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575329.686880:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575329.686883:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575329.686885:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575329.686888:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575329.686889:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575329.686891:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575329.686894:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575329.686896:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575329.686897:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575329.686899:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575329.686901:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686907:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575329.686910:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686914:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.686916:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575329.686918:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575329.686920:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575329.686922:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575329.686924:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306933433
00000100:00000001:3.0:1788575329.686925:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306934201
00000100:00000001:3.0:1788575329.686927:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306933433
00000100:00000001:3.0:1788575329.686929:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306934201
00000100:00000001:3.0:1788575329.686931:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575329.686932:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.686933:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575329.686935:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575329/real 1788575329]  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575329.686940:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575329.686941:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575329.686942:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575329.686944:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575329.686945:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575329.686947:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575329.686950:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575329.686959:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686967:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575329.686968:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.686969:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575329.686971:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575329.686976:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686984:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575329.686988:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.686996:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575329.686999:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.687001:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575329.687002:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.687004:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575329.687006:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575329.687009:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575329.687012:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575329.687015:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575329.687019:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575329.687026:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575329.687028:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.687030:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575329.687032:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393201408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.687036:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575329.687037:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575329.687039:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575329.687041:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575329.687042:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575329.687043:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575329.687044:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575329.687046:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:3.0:1788575329.687048:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575329.687049:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575329.687050:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:3.0:1788575329.687052:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575329.687053:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.687054:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575329.687056:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393201536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575329.687060:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575329.687061:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575329.687062:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575329.687064:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575329.687064:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575329.687065:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575329.687066:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575329.687067:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:3.0:1788575329.687068:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575329.687069:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575329.687070:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:3.0:1788575329.687072:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575329.687073:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.687074:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575329.687077:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575329.687078:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575329.687079:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575329.687080:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575330.069579:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575330.069589:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575330.069597:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575330.069598:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575330.069603:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575330.069604:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575330.069704:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575330.069710:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575330.069713:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575330.197498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575330.197506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575330.197508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575330.197512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575330.197516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575330.197521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575330.197524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575330.197526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575330.197528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575330.645495:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306934201, at 4306934208
00000100:00000040:1.0:1788575330.645495:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306934201, at 4306934208
00000100:00000010:0.0:1788575330.645520:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:1.0:1788575330.645523:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575331.093483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575331.093491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575331.093496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575331.093497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575331.093500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575331.093502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575331.093549:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575331.093553:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575331.093557:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575331.221506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575331.221515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575331.221517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575331.221523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575331.221530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575331.221534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575331.221537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575331.221539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575331.221542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575332.117573:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575332.117583:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575332.117591:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575332.117592:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575332.117597:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575332.117599:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575332.117666:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575332.117670:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575332.117673:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575332.245518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575332.245526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575332.245528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575332.245533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575332.245537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575332.245540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575332.245543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575332.245545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575332.245548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575333.141474:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575333.141482:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575333.141486:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575333.141487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575333.141490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575333.141491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575333.141556:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575333.141560:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575333.141564:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575333.269534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575333.269545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575333.269548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575333.269555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575333.269559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575333.269563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575333.269566:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575333.269568:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575333.269571:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575334.165550:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575334.165559:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575334.165566:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575334.165570:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575334.165574:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575334.165578:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575334.165581:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575334.165584:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575334.165586:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575334.165589:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575334.165593:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575334.165595:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575334.165597:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575334.165598:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575334.165600:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575334.165602:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575334.165603:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575334.165605:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575334.165673:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575334.165678:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575334.165683:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575334.293502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575334.293515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575334.293518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575334.293527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575334.293531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575334.293537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575334.293541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575334.293545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575334.293549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575334.805518:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575334.805532:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575334.805535:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575334.805541:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575334.805547:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575334.805555:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12265
00000400:00000200:1.0:1788575334.805566:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575334.805573:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575334.805578:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575334.805581:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805585:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575334.805588:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805591:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575334.805596:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575334.805599:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575334.805602:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575334.805606:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575334.805609:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575334.805612:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575334.805615:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805617:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575334.805620:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805622:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575334.805625:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575334.805629:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805633:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575334.805636:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575334.805648:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575334.805652:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575334.805655:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575334.805657:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575334.805671:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788575334.805676:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575334.805681:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575334.805683:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575334.805686:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575334.805690:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575334.805694:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575334.805703:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788575334.805712:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575334.805724:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575334.805736:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575334.805742:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575334.805747:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575334.805770:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575334.805773:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575334.805776:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575334.805778:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575334.805779:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575334.805781:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575334.805784:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12265
00000400:00000200:1.0:1788575334.805787:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575334.805789:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575334.805791:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575334.805793:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805795:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575334.805797:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805798:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575334.805800:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575334.805802:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575334.805803:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575334.805806:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575334.805807:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575334.805808:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575334.805811:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805812:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575334.805813:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805815:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575334.805816:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575334.805819:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575334.805819:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788575334.805824:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575334.805826:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575334.805827:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575334.805830:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1788575334.805832:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575334.805836:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575334.805839:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575334.805846:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575334.805852:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:1.0:1788575334.805853:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575334.805856:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1788575334.805857:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575334.805861:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575334.805862:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:1.0:1788575334.805864:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
02000000:00000001:3.0:1788575334.805866:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1788575334.805869:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575334.805870:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575334.805873:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575334.805876:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575334.805877:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393201664:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788575334.805880:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575334.805883:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1788575334.805884:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788575334.805886:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575334.805889:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:1.0:1788575334.805891:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
02000000:00000001:3.0:1788575334.805893:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575334.805895:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575334.805896:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:1.0:1788575334.805902:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:3.0:1788575334.805903:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575334.805907:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575334.805915:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575334.805918:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575334.805919:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575334.805924:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000040:1.0:1788575334.805925:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575334.805934:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575334.805934:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00080000:1.0:1788575334.805936:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575334.805940:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575334.805941:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575334.805943:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1788575334.805943:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393201664, portal 25
00000100:00000001:1.0:1788575334.805946:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:3.0:1788575334.805947:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575334.805950:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345471232 : -130483364080384 : ffff8953796f5500)
00000100:00080000:1.0:1788575334.805951:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12265
00000400:00000200:1.0:1788575334.805958:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575334.805961:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788575334.805964:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575334.805969:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575334.805975:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805979:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575334.805981:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788575334.805982:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.805985:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:3.0:1788575334.805989:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:1.0:1788575334.805990:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575334.805992:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575334.805995:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:3.0:1788575334.805995:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000020:00000040:1.0:1788575334.805997:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575334.806001:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575334.806003:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:3.0:1788575334.806005:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393201664, offset 0
00000100:00000040:1.0:1788575334.806006:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.806011:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575334.806013:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575334.806015:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575334.806018:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575334.806021:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788575334.806022:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575334.806026:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575334.806027:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575334.806029:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575334.806031:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575334.806032:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575334.806033:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575334.806039:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575334.806039:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575334.806041:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575334.806046:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788575334.806048:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575334.806049:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1788575334.806051:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:1.0:1788575334.806052:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575334.806056:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575334.806059:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575334.806060:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575334.806063:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000400:00000200:3.0:1788575334.806064:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1788575334.806066:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575334.806071:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575334.806074:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:1.0:1788575334.806081:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575334.806084:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1788575334.806085:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575334.806091:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575334.806115:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575334.806118:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575334.806121:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1788575334.806124:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:1.0:1788575334.806126:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12276)
00000400:00000200:3.0:1788575334.806141:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330831:330831:256:4294967295] 192.168.202.110@tcp LPNI seq info [330788:330788:8:4294967295]
00000400:00000200:3.0:1788575334.806149:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575334.806161:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575334.806168:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575334.806175:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575334.806183:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575334.806188:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788575334.806195:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575334.806199:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1788575334.806204:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575334.806208:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575334.806217:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575334.806222:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575334.806229:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575334.806231:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575334.806236:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575334.806239:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806248:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575334.806251:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806257:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575334.806259:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575334.806265:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575334.806269:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575334.806272:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306939321
00000100:00000010:3.0:1788575334.806277:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575334.806284:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575334.806290:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306939321
00000100:00000010:3.0:1788575334.806295:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575334.806298:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575334.806301:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575334.806303:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575334.806305:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575334.806308:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575334/real 1788575334]  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575334.806323:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575334.806324:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575334.806326:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575334.806328:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575334.806330:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575334.806333:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575334.806336:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575334.806346:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806354:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575334.806357:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.806359:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575334.806361:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575334.806366:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806375:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575334.806380:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806386:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575334.806388:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.806390:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575334.806413:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.806416:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575334.806417:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575334.806420:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575334.806424:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575334.806426:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575334.806430:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575334.806437:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393201664:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575334.806441:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.806443:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575334.806445:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5500 x1875444393201664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806449:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575334.806451:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575334.806454:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575334.806457:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575334.806458:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575334.806460:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575334.806461:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575334.806464:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:3.0:1788575334.806467:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575334.806469:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575334.806470:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5500.
00000100:00000001:3.0:1788575334.806473:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575334.806474:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.806476:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.806484:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575334.806486:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575334.806488:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575334.806489:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575334.806490:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575334.806493:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806498:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575334.806499:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575334.806501:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575334.806502:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575334.806504:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575334.806505:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201792:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575334.806508:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575334.806510:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575334.806511:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575334.806512:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575334.806515:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575334.806516:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575334.806520:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575334.806522:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575334.806525:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575334.806527:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393201792, portal 10
00000100:00000001:3.0:1788575334.806529:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575334.806530:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345469440 : -130483364082176 : ffff8953796f4e00)
00000100:00000040:3.0:1788575334.806535:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806540:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575334.806542:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575334.806543:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575334.806545:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393201792, offset 0
00000400:00000200:3.0:1788575334.806548:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575334.806551:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575334.806553:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575334.806556:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575334.806558:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575334.806561:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575334.806565:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575334.806568:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330832:330832:256:4294967295] 192.168.202.110@tcp LPNI seq info [330789:330789:8:4294967295]
00000400:00000200:3.0:1788575334.806573:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575334.806580:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575334.806585:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575334.806595:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575334.806600:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575334.806604:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788575334.806609:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575334.806614:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1788575334.806618:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575334.806620:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575334.806623:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575334.806627:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575334.806631:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575334.806633:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575334.806636:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575334.806641:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806649:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575334.806651:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806656:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575334.806657:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575334.806660:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575334.806663:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575334.806666:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575334.806667:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306938407
00000100:00000001:3.0:1788575334.806670:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306939321
00000100:00000001:3.0:1788575334.806672:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306938407
00000100:00000001:3.0:1788575334.806673:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306939321
00000100:00000001:3.0:1788575334.806675:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575334.806677:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575334.806678:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575334.806681:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575334/real 1788575334]  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575334.806685:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575334.806686:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575334.806723:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575334.806725:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575334.806726:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575334.806727:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575334.806730:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575334.806736:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806740:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575334.806742:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.806743:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575334.806745:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575334.806749:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806755:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575334.806760:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806767:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575334.806770:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.806772:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575334.806773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.806775:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575334.806777:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575334.806780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575334.806784:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575334.806786:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575334.806791:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575334.806800:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201792:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575334.806805:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575334.806808:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806812:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575334.806814:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575334.806815:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575334.806817:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575334.806818:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575334.806820:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201920:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575334.806823:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575334.806824:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575334.806826:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575334.806827:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575334.806830:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575334.806831:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575334.806835:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575334.806836:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575334.806839:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575334.806842:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393201920, portal 4
00000100:00000001:3.0:1788575334.806844:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575334.806845:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345481088 : -130483364070528 : ffff8953796f7b80)
00000100:00000040:3.0:1788575334.806849:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806854:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575334.806856:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575334.806857:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575334.806859:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393201920, offset 0
00000400:00000200:3.0:1788575334.806861:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575334.806864:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575334.806867:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575334.806869:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575334.806871:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575334.806873:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575334.806875:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575334.806877:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330833:330833:256:4294967295] 192.168.202.110@tcp LPNI seq info [330790:330790:8:4294967295]
00000400:00000200:3.0:1788575334.806880:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575334.806884:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575334.806887:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575334.806889:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575334.806892:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575334.806894:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788575334.806899:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575334.806902:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1788575334.806907:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575334.806909:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575334.806912:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575334.806916:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575334.806920:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575334.806922:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575334.806924:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575334.806928:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806937:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575334.806941:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.806948:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575334.806950:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575334.806954:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575334.806957:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575334.806961:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575334.806964:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306938553
00000100:00000001:3.0:1788575334.806967:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306939321
00000100:00000001:3.0:1788575334.806969:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306938553
00000100:00000001:3.0:1788575334.806972:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306939321
00000100:00000001:3.0:1788575334.806975:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575334.806977:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575334.806979:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575334.806983:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575334/real 1788575334]  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575334.806992:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575334.806994:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575334.806996:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575334.806999:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575334.807000:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575334.807003:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575334.807006:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575334.807015:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.807023:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575334.807025:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.807027:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575334.807029:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575334.807033:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.807041:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575334.807045:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.807050:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575334.807052:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.807053:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575334.807054:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.807056:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575334.807058:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575334.807060:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575334.807062:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575334.807064:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575334.807067:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575334.807072:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393201920:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575334.807075:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.807076:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575334.807078:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4e00 x1875444393201792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.807082:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575334.807083:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575334.807085:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575334.807087:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575334.807088:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575334.807089:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575334.807091:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575334.807092:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:3.0:1788575334.807110:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575334.807112:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575334.807113:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4e00.
00000100:00000001:3.0:1788575334.807115:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575334.807116:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.807117:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575334.807120:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7b80 x1875444393201920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575334.807125:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575334.807126:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575334.807127:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575334.807129:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575334.807130:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575334.807131:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575334.807133:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575334.807134:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:3.0:1788575334.807136:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575334.807136:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575334.807137:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7b80.
00000100:00000001:3.0:1788575334.807139:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575334.807139:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.807140:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575334.807143:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575334.807144:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575334.807145:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575334.807146:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575335.189506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575335.189513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575335.189519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575335.189520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575335.189524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575335.189526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575335.189577:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575335.189583:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575335.189588:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575335.317553:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575335.317563:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575335.317565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575335.317571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575335.317575:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575335.317579:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575335.317581:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575335.317583:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575335.317586:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575335.765535:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306939321, at 4306939328
00000100:00000040:1.0:1788575335.765535:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306939321, at 4306939328
00000100:00000010:1.0:1788575335.765555:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788575335.765561:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575336.213503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575336.213509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575336.213513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575336.213514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575336.213516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575336.213517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575336.213605:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575336.213612:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575336.213617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575336.341490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575336.341497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575336.341499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575336.341503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575336.341506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575336.341509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575336.341511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575336.341513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575336.341515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575337.237512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575337.237520:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575337.237523:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575337.237526:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575337.237532:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575337.237534:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575337.237537:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575337.237540:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575337.237544:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575337.237546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575337.237550:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575337.237552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575337.237616:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575337.237627:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575337.237631:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575337.365516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575337.365523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575337.365526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575337.365532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575337.365535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575337.365539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575337.365548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575337.365550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575337.365554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575338.261494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575338.261504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575338.261509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575338.261510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575338.261512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575338.261513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575338.261567:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575338.261570:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575338.261573:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575338.389488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575338.389494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575338.389496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575338.389500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575338.389504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575338.389507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575338.389510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575338.389512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575338.389514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575339.285475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575339.285483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575339.285489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575339.285491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575339.285495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575339.285496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575339.285554:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575339.285557:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575339.285560:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575339.413500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575339.413507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575339.413510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575339.413515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575339.413519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575339.413523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575339.413526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575339.413529:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575339.413533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575339.925559:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575339.925574:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575339.925578:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575339.925583:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575339.925589:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575339.925599:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12271
00000400:00000200:3.0:1788575339.925612:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575339.925619:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575339.925623:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575339.925627:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575339.925632:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575339.925634:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575339.925638:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575339.925643:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575339.925645:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575339.925649:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575339.925653:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575339.925656:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575339.925658:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575339.925662:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575339.925665:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575339.925667:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575339.925669:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575339.925673:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575339.925677:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575339.925681:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575339.925684:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575339.925689:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575339.925692:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575339.925695:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575339.925698:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575339.925710:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1180.
00000020:00000040:3.0:1788575339.925715:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575339.925720:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575339.925722:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575339.925726:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575339.925730:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575339.925735:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575339.925744:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575339.925754:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575339.925765:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575339.925777:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575339.925783:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575339.925789:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca1dca9a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575339.925822:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575339.925825:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575339.925829:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575339.925830:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575339.925832:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575339.925835:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575339.925839:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12271
00000400:00000200:3.0:1788575339.925844:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575339.925851:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575339.925854:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575339.925857:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575339.925860:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575339.925862:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575339.925865:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575339.925871:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575339.925873:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575339.925876:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575339.925878:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575339.925881:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575339.925885:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575339.925891:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575339.925891:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575339.925904:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575339.925916:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575339.925918:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575339.925921:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:3.0:1788575339.925922:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575339.925925:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575339.925931:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575339.925934:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575339.925941:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00100000:2.0:1788575339.925946:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393202048:192.168.202.110@tcp:250:kworker.0
00000100:00000040:3.0:1788575339.925947:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575339.925964:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575339.925964:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575339.925975:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575339.925976:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575339.925981:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575339.925985:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:3.0:1788575339.925987:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575339.925992:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575339.925996:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575339.925997:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575339.926002:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575339.926009:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575339.926011:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575339.926032:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
02000000:00000001:3.0:1788575339.926035:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575339.926043:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000001:3.0:1788575339.926043:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575339.926054:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000200:2.0:1788575339.926055:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393202048, portal 25
00000100:00000001:2.0:1788575339.926061:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575339.926065:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000008:00000001:3.0:1788575339.926070:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575339.926071:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575339.926080:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.926094:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575339.926097:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575339.926099:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575339.926103:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393202048, offset 0
00000100:00000010:3.0:1788575339.926104:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000020:00000040:3.0:1788575339.926109:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:2.0:1788575339.926112:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575339.926121:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575339.926126:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575339.926134:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575339.926136:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575339.926140:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575339.926143:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:3.0:1788575339.926145:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575339.926151:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575339.926156:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000010:3.0:1788575339.926157:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000400:00000200:2.0:1788575339.926163:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575339.926167:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575339.926173:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575339.926182:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330834:330834:256:4294967295] 192.168.202.110@tcp LPNI seq info [330791:330791:8:4294967295]
00000100:00100000:3.0:1788575339.926182:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575339.926204:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575339.926214:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575339.926217:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575339.926228:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575339.926236:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575339.926256:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575339.926266:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000100:00000001:3.0:1788575339.926268:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575339.926274:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1788575339.926277:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575339.926285:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00080000:3.0:1788575339.926287:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575339.926290:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575339.926292:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575339.926297:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:2.1:1788575339.926302:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1788575339.926308:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12271
00000800:00000200:2.1:1788575339.926309:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575339.926327:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575339.926328:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:2.0:1788575339.926334:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:3.0:1788575339.926340:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575339.926341:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:3.0:1788575339.926349:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575339.926354:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575339.926360:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000100:00000040:3.0:1788575339.926360:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:2.0:1788575339.926367:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:3.0:1788575339.926371:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575339.926373:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:3.0:1788575339.926378:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575339.926380:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575339.926388:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575339.926431:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575339.926437:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.926447:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.926455:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000001:3.0:1788575339.926456:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575339.926460:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575339.926465:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575339.926467:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575339.926469:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575339.926475:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575339.926478:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1788575339.926483:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788575339.926484:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306944442
00000100:00000010:2.0:1788575339.926495:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c5a0.
00000100:00000040:3.0:1788575339.926496:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575339.926503:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575339.926505:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:3.0:1788575339.926511:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575339.926517:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575339.926520:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788575339.926524:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306944442
00000100:00080000:3.0:1788575339.926525:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575339.926531:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575339.926533:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c060.
02000000:00000001:3.0:1788575339.926535:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575339.926545:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575339.926546:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:3.0:1788575339.926551:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.926555:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575339.926557:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575339.926561:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575339.926562:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575339.926567:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000008:00000001:3.0:1788575339.926574:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575339.926579:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575339/real 1788575339]  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575339.926580:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575339.926598:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000400:00000001:2.0:1788575339.926600:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575339.926607:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575339.926609:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000020:00000040:3.0:1788575339.926609:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575339.926619:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
02000000:00000001:3.0:1788575339.926619:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1788575339.926625:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
02000000:00000001:3.0:1788575339.926626:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.926632:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000001:3.0:1788575339.926634:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575339.926639:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1788575339.926640:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575339.926647:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575339.926659:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788575339.926660:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:3.0:1788575339.926670:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.926672:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575339.926677:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.926680:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575339.926684:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575339.926686:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:2.0:1788575339.926692:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575339.926738:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575339.926740:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575339.926745:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575339.926752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575339.926759:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575339.926811:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.926812:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575339.926821:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575339.926821:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575339.926832:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575339.926835:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575339.926836:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575339.926839:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575339.926845:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575339.926848:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:3.0:1788575339.926849:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12281)
00000100:00080000:2.0:1788575339.926855:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575339.926858:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575339.926863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575339.926872:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393202048:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575339.926877:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.926881:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575339.926883:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393202048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.926893:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575339.926895:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575339.926898:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575339.926900:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575339.926902:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575339.926904:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575339.926905:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575339.926908:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575339.926910:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575339.926911:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575339.926913:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:2.0:1788575339.926915:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575339.926916:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.926918:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.926924:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575339.926925:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.926927:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575339.926929:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575339.926930:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575339.926933:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.926938:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575339.926939:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575339.926941:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575339.926942:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575339.926944:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575339.926946:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202176:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575339.926948:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575339.926950:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575339.926951:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575339.926953:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575339.926955:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575339.926957:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575339.926959:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575339.926962:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575339.926965:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575339.926967:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393202176, portal 10
00000100:00000001:2.0:1788575339.926969:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575339.926970:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:2.0:1788575339.926975:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.926980:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575339.926981:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575339.926983:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575339.926984:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393202176, offset 0
00000400:00000200:2.0:1788575339.926987:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575339.926990:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575339.926993:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575339.926995:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575339.926997:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575339.926999:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575339.927000:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575339.927003:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330835:330835:256:4294967295] 192.168.202.110@tcp LPNI seq info [330792:330792:8:4294967295]
00000400:00000200:2.0:1788575339.927005:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575339.927011:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575339.927014:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575339.927030:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575339.927034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575339.927036:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575339.927039:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575339.927041:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575339.927044:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575339.927045:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575339.927047:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575339.927050:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575339.927052:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575339.927054:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575339.927056:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575339.927058:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927064:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575339.927066:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927071:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.927072:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575339.927074:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575339.927077:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575339.927080:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575339.927081:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306943527
00000100:00000001:2.0:1788575339.927083:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306944442
00000100:00000001:2.0:1788575339.927085:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306943527
00000100:00000001:2.0:1788575339.927086:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306944442
00000100:00000001:2.0:1788575339.927088:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575339.927089:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.927091:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575339.927093:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575339/real 1788575339]  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575339.927098:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575339.927099:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575339.927100:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575339.927101:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575339.927103:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575339.927104:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575339.927106:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575339.927111:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927120:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575339.927121:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927122:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575339.927123:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575339.927125:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927129:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575339.927132:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927136:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575339.927137:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927138:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575339.927139:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927140:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575339.927142:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575339.927144:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575339.927146:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575339.927148:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575339.927150:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575339.927155:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202176:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575339.927157:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575339.927160:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927164:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575339.927165:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575339.927167:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575339.927168:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575339.927170:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575339.927172:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575339.927174:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575339.927176:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575339.927178:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575339.927179:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575339.927183:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575339.927185:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575339.927188:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575339.927191:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575339.927194:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575339.927197:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393202304, portal 4
00000100:00000001:2.0:1788575339.927199:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575339.927200:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:2.0:1788575339.927207:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927214:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575339.927217:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575339.927219:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575339.927222:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393202304, offset 0
00000400:00000200:2.0:1788575339.927226:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575339.927230:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575339.927234:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575339.927237:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575339.927239:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575339.927240:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575339.927242:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575339.927244:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330836:330836:256:4294967295] 192.168.202.110@tcp LPNI seq info [330793:330793:8:4294967295]
00000400:00000200:2.0:1788575339.927247:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575339.927252:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575339.927256:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575339.927258:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575339.927261:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575339.927263:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575339.927266:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575339.927267:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575339.927270:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575339.927271:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575339.927274:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575339.927276:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575339.927278:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575339.927280:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575339.927281:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575339.927284:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927290:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575339.927292:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927296:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.927297:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575339.927299:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575339.927301:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575339.927304:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575339.927305:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306943673
00000100:00000001:2.0:1788575339.927307:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306944442
00000100:00000001:2.0:1788575339.927308:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306943673
00000100:00000001:2.0:1788575339.927310:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306944442
00000100:00000001:2.0:1788575339.927311:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575339.927312:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.927313:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575339.927316:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575339/real 1788575339]  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575339.927321:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575339.927322:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575339.927324:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575339.927325:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575339.927326:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575339.927327:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575339.927329:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575339.927335:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927339:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575339.927340:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927341:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575339.927342:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575339.927344:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927348:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575339.927350:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927354:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575339.927356:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927357:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575339.927357:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927359:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575339.927360:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575339.927361:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575339.927364:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575339.927365:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575339.927368:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575339.927373:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575339.927375:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927377:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575339.927379:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393202176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927382:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575339.927383:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575339.927385:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575339.927386:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575339.927387:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575339.927388:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575339.927389:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575339.927409:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575339.927412:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575339.927413:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575339.927414:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:2.0:1788575339.927417:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575339.927418:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927420:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575339.927424:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393202304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575339.927431:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575339.927432:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575339.927434:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575339.927436:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575339.927438:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575339.927439:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575339.927441:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575339.927443:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575339.927445:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575339.927447:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575339.927448:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:2.0:1788575339.927452:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575339.927453:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927455:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575339.927459:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575339.927460:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575339.927462:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575339.927464:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575340.309556:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575340.309566:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575340.309574:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575340.309576:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575340.309580:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575340.309582:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575340.309677:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575340.309683:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575340.309686:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575340.437539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575340.437549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575340.437551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575340.437557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575340.437561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575340.437565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575340.437568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575340.437570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575340.437573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575340.885521:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306944442, at 4306944448
00000100:00000040:1.0:1788575340.885521:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306944442, at 4306944448
00000100:00000010:0.0:1788575340.885544:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000100:00000010:1.0:1788575340.885546:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00010000:00000001:3.0:1788575341.333468:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575341.333475:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575341.333482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575341.333483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575341.333487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575341.333489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575341.333583:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575341.333590:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575341.333595:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575341.461533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575341.461542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575341.461544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575341.461549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575341.461553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575341.461557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575341.461560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575341.461562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575341.461565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575342.357498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575342.357504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575342.357507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575342.357508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575342.357511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575342.357512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575342.357588:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575342.357593:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575342.357598:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575342.485495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575342.485501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575342.485503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575342.485506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575342.485509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575342.485512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575342.485515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575342.485517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575342.485519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575343.381523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575343.381528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575343.381532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575343.381533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575343.381535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575343.381537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575343.381600:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575343.381604:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575343.381607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575343.509508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575343.509513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575343.509515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575343.509518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575343.509521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575343.509524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575343.509526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575343.509528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575343.509531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575344.405507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575344.405514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575344.405520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575344.405523:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575344.405525:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575344.405529:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575344.405532:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575344.405534:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575344.405536:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575344.405539:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575344.405543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575344.405544:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575344.405546:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575344.405547:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575344.405549:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575344.405550:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575344.405552:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575344.405554:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575344.405604:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575344.405607:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575344.405610:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575344.533510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575344.533518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575344.533521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575344.533526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575344.533541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575344.533545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575344.533549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575344.533552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575344.533556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575345.045539:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575345.045551:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575345.045553:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575345.045558:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575345.045562:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575345.045570:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12276
00000400:00000200:2.0:1788575345.045579:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575345.045586:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575345.045590:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575345.045593:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.045597:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575345.045599:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.045603:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575345.045607:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575345.045610:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575345.045613:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575345.045617:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575345.045620:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575345.045622:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575345.045625:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.045627:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575345.045630:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.045632:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575345.045635:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575345.045639:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575345.045642:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575345.045645:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575345.045649:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575345.045652:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575345.045654:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575345.045657:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575345.045666:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01180.
00000020:00000040:2.0:1788575345.045671:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575345.045675:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575345.045677:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575345.045680:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575345.045694:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575345.045698:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575345.045707:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:2.0:1788575345.045713:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575345.045728:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575345.045738:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575345.045743:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575345.045748:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b74282d7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575345.045773:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575345.045776:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575345.045781:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575345.045783:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575345.045785:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575345.045789:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575345.045792:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12276
00000400:00000200:2.0:1788575345.045797:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575345.045801:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575345.045804:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575345.045807:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575345.045809:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575345.045810:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575345.045816:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1788575345.045817:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.045822:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575345.045822:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788575345.045827:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575345.045829:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575345.045832:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:3.0:1788575345.045833:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788575345.045835:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575345.045839:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575345.045844:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575345.045852:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575345.045854:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575345.045857:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575345.045859:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575345.045863:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575345.045866:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1788575345.045867:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575345.045872:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575345.045875:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575345.045877:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575345.045879:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575345.045885:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575345.045887:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575345.045888:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575345.045893:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575345.045894:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575345.045900:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00100000:3.0:1788575345.045901:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393202432:192.168.202.110@tcp:250:kworker.0
00000008:00000001:2.0:1788575345.045907:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575345.045910:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1788575345.045912:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575345.045917:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1788575345.045918:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03480.
02000000:00000001:3.0:1788575345.045922:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575345.045923:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575345.045928:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1788575345.045930:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575345.045932:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575345.045934:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575345.045935:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575345.045954:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575345.045959:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000100:00000001:2.0:1788575345.045964:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575345.045972:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:3.0:1788575345.045972:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575345.045978:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575345.045987:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575345.045989:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575345.045992:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575345.045996:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000040:2.0:1788575345.046000:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9656041] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788575345.046006:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000001:2.0:1788575345.046015:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575345.046016:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393202432, portal 25
00000100:00080000:2.0:1788575345.046018:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575345.046021:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1788575345.046024:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575345.046025:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000001:2.0:1788575345.046028:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575345.046032:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:3.0:1788575345.046037:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575345.046038:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575345.046044:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12276
00000400:00000200:2.0:1788575345.046054:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575345.046060:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575345.046066:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575345.046073:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575345.046074:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.046080:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575345.046081:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575345.046086:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575345.046092:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393202432, offset 0
00000100:00000040:2.0:1788575345.046096:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.046103:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575345.046104:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:2.0:1788575345.046109:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575345.046111:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575345.046113:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575345.046117:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575345.046118:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1788575345.046122:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575345.046126:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575345.046129:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1788575345.046133:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.046137:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575345.046139:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1788575345.046142:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.046146:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575345.046147:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:2.0:1788575345.046152:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575345.046156:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1788575345.046159:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575345.046164:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575345.046165:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575345.046169:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575345.046174:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575345.046174:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330837:330837:256:4294967295] 192.168.202.110@tcp LPNI seq info [330794:330794:8:4294967295]
02000000:00000001:2.0:1788575345.046179:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575345.046183:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575345.046184:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:2.0:1788575345.046187:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575345.046192:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575345.046196:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575345.046202:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000010:2.0:1788575345.046204:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000020:00000040:2.0:1788575345.046209:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:3.0:1788575345.046214:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788575345.046215:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575345.046219:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575345.046222:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:3.0:1788575345.046222:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
02000000:00000001:2.0:1788575345.046224:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575345.046228:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575345.046233:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:2.0:1788575345.046234:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000800:00000200:3.0:1788575345.046240:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1788575345.046241:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575345.046249:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1788575345.046251:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575345.046258:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575345.046269:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575345.046275:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000100:00000001:2.0:1788575345.046277:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575345.046282:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575345.046284:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1788575345.046289:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575345.046295:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1788575345.046306:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575345.046306:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575345.046311:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000040:2.0:1788575345.046312:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575345.046318:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:2.0:1788575345.046319:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575345.046326:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12286)
00000100:00000200:3.0:1788575345.046327:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046341:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575345.046345:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575370 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046360:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575345.046363:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575345.046369:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575345.046374:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575345.046377:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306949561
00000100:00000010:3.0:1788575345.046383:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575345.046389:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575345.046417:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306949562
00000100:00000010:3.0:1788575345.046422:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575345.046426:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575345.046429:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575345.046432:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575345.046435:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575345.046440:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575345/real 1788575345]  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575345.046452:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575345.046454:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575345.046456:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575345.046459:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575345.046461:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575345.046463:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575345.046467:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575345.046477:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046485:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575345.046486:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.046489:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575345.046492:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575345.046496:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046505:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575345.046510:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046518:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575345.046522:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.046524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575345.046525:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.046528:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575345.046530:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575345.046534:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575345.046538:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575345.046540:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575345.046546:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575345.046555:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393202432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575345.046561:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.046563:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575345.046566:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393202432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046573:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575345.046576:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575345.046579:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575345.046583:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575345.046585:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575345.046587:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575345.046590:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575345.046592:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575345.046597:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575345.046600:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575345.046602:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:3.0:1788575345.046605:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575345.046616:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.046619:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.046626:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575345.046628:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575345.046630:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575345.046632:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575345.046633:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575345.046637:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046646:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575345.046647:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575345.046649:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575345.046651:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575345.046654:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575345.046657:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575345.046661:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575345.046663:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575345.046664:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575345.046666:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575345.046670:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575345.046672:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575345.046676:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575345.046679:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575345.046683:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575345.046687:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393202560, portal 10
00000100:00000001:3.0:1788575345.046689:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575345.046690:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:3.0:1788575345.046697:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046705:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575345.046708:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575345.046711:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575345.046714:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393202560, offset 0
00000400:00000200:3.0:1788575345.046719:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575345.046724:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575345.046728:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575345.046731:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575345.046734:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575345.046736:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575345.046739:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575345.046743:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330838:330838:256:4294967295] 192.168.202.110@tcp LPNI seq info [330795:330795:8:4294967295]
00000400:00000200:3.0:1788575345.046747:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575345.046755:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575345.046760:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575345.046764:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575345.046768:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575345.046771:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575345.046777:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575345.046780:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575345.046785:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575345.046787:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575345.046790:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575345.046794:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575345.046798:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575345.046800:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575345.046803:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575345.046807:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046816:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575345.046819:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575370 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046826:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575345.046828:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575345.046832:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575345.046836:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575345.046840:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575345.046842:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306948647
00000100:00000001:3.0:1788575345.046845:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306949561
00000100:00000001:3.0:1788575345.046848:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306948647
00000100:00000001:3.0:1788575345.046850:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306949562
00000100:00000001:3.0:1788575345.046853:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575345.046855:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575345.046857:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575345.046862:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575345/real 1788575345]  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575345.046871:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575345.046873:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575345.046875:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575345.046878:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575345.046879:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575345.046882:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575345.046886:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575345.046895:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046902:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575345.046903:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.046905:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575345.046906:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575345.046910:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046917:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575345.046920:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046927:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575345.046930:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.046931:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575345.046933:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.046934:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575345.046937:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575345.046953:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575345.046957:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575345.046959:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575345.046963:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575345.046971:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575345.046975:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575345.046979:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.046985:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575345.046987:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575345.046989:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575345.046991:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575345.046993:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575345.046996:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202688:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575345.046999:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575345.047001:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575345.047003:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575345.047004:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575345.047008:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575345.047010:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575345.047014:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575345.047018:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575345.047021:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575345.047024:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393202688, portal 4
00000100:00000001:3.0:1788575345.047026:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575345.047027:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575345.047033:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.047040:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575345.047043:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575345.047045:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575345.047047:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393202688, offset 0
00000400:00000200:3.0:1788575345.047051:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575345.047055:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575345.047059:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575345.047062:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575345.047065:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575345.047067:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575345.047069:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575345.047073:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330839:330839:256:4294967295] 192.168.202.110@tcp LPNI seq info [330796:330796:8:4294967295]
00000400:00000200:3.0:1788575345.047077:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575345.047083:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575345.047088:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575345.047091:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575345.047095:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575345.047097:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575345.047101:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575345.047104:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575345.047107:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575345.047109:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575345.047112:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575345.047116:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575345.047119:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575345.047121:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575345.047124:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575345.047127:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.047135:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575345.047138:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575370 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.047144:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575345.047146:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575345.047149:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575345.047152:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575345.047155:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575345.047157:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306948793
00000100:00000001:3.0:1788575345.047160:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306949561
00000100:00000001:3.0:1788575345.047162:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306948793
00000100:00000001:3.0:1788575345.047164:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306949562
00000100:00000001:3.0:1788575345.047166:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575345.047168:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575345.047170:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575345.047173:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575345/real 1788575345]  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575345.047181:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575345.047182:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575345.047184:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575345.047186:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575345.047188:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575345.047190:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575345.047193:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575345.047201:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.047212:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575345.047214:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.047216:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575345.047217:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575345.047221:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.047228:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575345.047231:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.047238:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575345.047240:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.047242:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575345.047244:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.047245:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575345.047247:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575345.047249:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575345.047253:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575345.047255:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575345.047259:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575345.047267:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202688:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575345.047271:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.047273:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575345.047276:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393202560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.047282:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575345.047284:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575345.047286:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575345.047288:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575345.047290:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575345.047291:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575345.047293:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575345.047295:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575345.047298:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575345.047299:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575345.047300:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:3.0:1788575345.047303:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575345.047304:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.047306:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575345.047309:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393202688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575370 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575345.047315:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575345.047317:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575345.047318:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575345.047320:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575345.047321:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575345.047322:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575345.047325:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575345.047326:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575345.047328:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575345.047329:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575345.047331:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575345.047333:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575345.047335:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.047337:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575345.047340:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575345.047341:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575345.047344:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575345.047345:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575345.429504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575345.429509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575345.429513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575345.429514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575345.429516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575345.429518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575345.429576:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575345.429581:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575345.429584:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575345.557533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575345.557541:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575345.557544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575345.557550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575345.557554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575345.557557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575345.557560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575345.557563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575345.557565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575346.005500:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306949561, at 4306949568
00000100:00000040:1.0:1788575346.005500:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306949562, at 4306949568
00000100:00000010:1.0:1788575346.005518:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575346.005520:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575346.453487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575346.453494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575346.453501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575346.453502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575346.453506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575346.453507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575346.453556:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575346.453560:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575346.453564:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575346.581449:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575346.581456:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575346.581457:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575346.581462:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575346.581465:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575346.581469:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575346.581471:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575346.581473:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575346.581475:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575347.477466:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575347.477474:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575347.477477:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575347.477480:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575347.477485:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575347.477487:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575347.477489:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575347.477492:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575347.477496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575347.477497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575347.477501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575347.477502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575347.477561:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575347.477565:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575347.477569:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575347.605490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575347.605496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575347.605498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575347.605502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575347.605505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575347.605507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575347.605509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575347.605511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575347.605513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575348.501472:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575348.501479:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575348.501485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575348.501487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575348.501490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575348.501492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575348.501551:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575348.501554:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575348.501557:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575348.629533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575348.629541:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575348.629543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575348.629548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575348.629551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575348.629554:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575348.629557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575348.629559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575348.629562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575349.525495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575349.525502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575349.525510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575349.525512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575349.525515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575349.525517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575349.525565:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575349.525568:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575349.525571:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575349.653472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575349.653479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575349.653480:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575349.653485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575349.653488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575349.653490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575349.653493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575349.653495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575349.653497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575350.165484:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575350.165494:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575350.165497:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575350.165502:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575350.165506:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575350.165513:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12281
00000400:00000200:3.0:1788575350.165523:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575350.165530:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575350.165533:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575350.165535:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165538:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575350.165540:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165541:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575350.165544:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575350.165546:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575350.165548:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575350.165551:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575350.165553:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575350.165554:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575350.165557:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165558:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575350.165559:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165560:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575350.165563:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575350.165566:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165567:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575350.165570:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575350.165572:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575350.165574:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575350.165576:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575350.165577:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575350.165585:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788575350.165587:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575350.165590:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575350.165591:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575350.165594:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575350.165596:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575350.165599:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575350.165614:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8c00.
00000100:00000001:3.0:1788575350.165620:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575350.165627:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575350.165635:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575350.165638:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575350.165641:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575350.165660:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575350.165663:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575350.165666:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575350.165667:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575350.165668:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575350.165671:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575350.165673:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12281
00000400:00000200:3.0:1788575350.165676:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575350.165679:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575350.165681:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575350.165685:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165686:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575350.165689:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575350.165689:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575350.165694:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575350.165694:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575350.165697:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575350.165698:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788575350.165699:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575350.165702:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575350.165704:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000040:2.0:1788575350.165707:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1788575350.165707:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575350.165709:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575350.165717:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165723:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575350.165724:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575350.165727:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575350.165727:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575350.165732:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575350.165732:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575350.165735:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1788575350.165736:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575350.165740:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575350.165742:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:2.0:1788575350.165747:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393202816:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575350.165748:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575350.165753:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575350.165756:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575350.165759:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575350.165760:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575350.165764:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575350.165765:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575350.165770:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575350.165771:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:3.0:1788575350.165776:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1788575350.165778:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
00000008:00000001:3.0:1788575350.165779:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575350.165783:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575350.165784:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575350.165791:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575350.165794:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:2.0:1788575350.165796:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575350.165797:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575350.165800:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575350.165802:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575350.165803:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1788575350.165805:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
02000000:00000001:3.0:1788575350.165805:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575350.165813:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000100:00000001:3.0:1788575350.165816:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575350.165817:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00100000:3.0:1788575350.165822:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788575350.165827:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393202816, portal 25
00000100:00000001:3.0:1788575350.165829:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575350.165833:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575350.165833:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575350.165838:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:3.0:1788575350.165839:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575350.165849:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575350.165851:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575350.165852:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575350.165858:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575350.165875:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575350.165877:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575350.165879:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575350.165882:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12281
00000400:00000200:3.0:1788575350.165885:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575350.165888:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575350.165889:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575350.165891:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165893:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575350.165894:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165896:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575350.165898:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575350.165899:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575350.165900:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575350.165901:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575350.165902:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575350.165904:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575350.165906:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165907:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575350.165908:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165909:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575350.165910:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575350.165912:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165913:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575350.165915:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575350.165917:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575350.165918:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575350.165920:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575350.165921:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575350.165923:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575350.165925:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575350.165927:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000020:00000040:3.0:1788575350.165929:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575350.165931:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575350.165932:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575350.165933:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575350.165934:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575350.165935:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575350.165940:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:3.0:1788575350.165944:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575350.165950:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575350.165955:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575350.165962:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575350.165967:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1788575350.165968:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000001:3.0:1788575350.165972:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575350.165974:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393202816, offset 0
00000100:00000040:3.0:1788575350.165981:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575350.165983:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575350.165994:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575350.165997:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575350.166000:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575350.166003:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1788575350.166005:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575350.166011:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575350.166014:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575350.166017:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575350.166020:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:3.0:1788575350.166021:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12291)
00000400:00000200:2.0:1788575350.166025:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330840:330840:256:4294967295] 192.168.202.110@tcp LPNI seq info [330797:330797:8:4294967295]
00000400:00000200:2.0:1788575350.166030:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575350.166038:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575350.166044:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575350.166048:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575350.166053:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575350.166056:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575350.166061:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575350.166063:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575350.166066:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575350.166068:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575350.166073:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575350.166077:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575350.166081:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575350.166083:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575350.166086:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575350.166090:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166098:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575350.166100:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575375 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166105:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575350.166106:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575350.166110:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575350.166113:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575350.166116:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306954681
00000100:00000010:2.0:1788575350.166120:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c180.
00000100:00000001:2.0:1788575350.166126:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575350.166130:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306954681
00000100:00000010:2.0:1788575350.166132:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cea0.
00000100:00000001:2.0:1788575350.166134:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575350.166137:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575350.166139:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575350.166141:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575350.166144:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575350/real 1788575350]  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575350.166151:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575350.166153:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575350.166155:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575350.166157:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575350.166158:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575350.166159:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575350.166162:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575350.166167:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166181:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575350.166182:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166184:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575350.166186:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575350.166188:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166193:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575350.166196:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166201:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575350.166203:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166204:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575350.166205:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166207:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575350.166209:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575350.166211:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575350.166213:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575350.166215:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575350.166217:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575350.166223:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393202816:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575350.166226:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166228:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575350.166230:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393202816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166234:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575350.166237:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575350.166238:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575350.166241:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575350.166242:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575350.166243:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575350.166245:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575350.166247:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:2.0:1788575350.166250:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575350.166251:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575350.166252:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788575350.166254:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575350.166255:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166257:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166263:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575350.166264:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575350.166266:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575350.166267:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575350.166268:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575350.166271:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166276:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575350.166277:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575350.166278:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575350.166279:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575350.166281:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575350.166283:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202944:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575350.166285:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575350.166286:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575350.166288:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575350.166289:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575350.166291:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575350.166292:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575350.166295:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575350.166297:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575350.166299:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575350.166301:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393202944, portal 10
00000100:00000001:2.0:1788575350.166303:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575350.166304:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:2.0:1788575350.166309:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166313:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575350.166315:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575350.166316:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575350.166318:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393202944, offset 0
00000400:00000200:2.0:1788575350.166321:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575350.166323:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575350.166326:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575350.166328:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575350.166330:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575350.166331:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575350.166333:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575350.166335:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330841:330841:256:4294967295] 192.168.202.110@tcp LPNI seq info [330798:330798:8:4294967295]
00000400:00000200:2.0:1788575350.166338:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575350.166343:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575350.166346:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575350.166348:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575350.166351:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575350.166353:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575350.166356:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575350.166358:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575350.166360:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575350.166362:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575350.166364:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575350.166366:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575350.166368:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575350.166369:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575350.166371:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575350.166373:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166379:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575350.166381:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575375 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166385:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575350.166386:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575350.166388:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575350.166408:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575350.166411:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575350.166412:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306953767
00000100:00000001:2.0:1788575350.166415:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306954681
00000100:00000001:2.0:1788575350.166416:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306953767
00000100:00000001:2.0:1788575350.166418:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306954681
00000100:00000001:2.0:1788575350.166420:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575350.166421:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575350.166423:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575350.166426:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575350/real 1788575350]  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575350.166431:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575350.166432:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575350.166434:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575350.166435:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575350.166436:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575350.166437:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575350.166439:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575350.166448:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166455:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575350.166457:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166459:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575350.166460:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575350.166464:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166472:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575350.166476:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166483:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575350.166486:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166488:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575350.166489:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166491:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575350.166493:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575350.166496:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575350.166500:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575350.166502:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575350.166507:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575350.166516:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393202944:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575350.166520:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575350.166524:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166531:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575350.166533:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575350.166535:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575350.166537:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575350.166539:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575350.166543:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203072:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575350.166546:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575350.166548:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575350.166550:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575350.166552:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575350.166555:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575350.166558:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575350.166561:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575350.166563:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575350.166566:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575350.166568:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393203072, portal 4
00000100:00000001:2.0:1788575350.166569:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575350.166570:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:2.0:1788575350.166576:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166584:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575350.166586:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575350.166587:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575350.166589:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393203072, offset 0
00000400:00000200:2.0:1788575350.166591:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575350.166594:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575350.166597:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575350.166599:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575350.166601:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575350.166603:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575350.166604:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575350.166607:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330842:330842:256:4294967295] 192.168.202.110@tcp LPNI seq info [330799:330799:8:4294967295]
00000400:00000200:2.0:1788575350.166609:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575350.166614:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575350.166618:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575350.166620:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575350.166623:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575350.166625:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575350.166627:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575350.166629:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575350.166633:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575350.166634:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575350.166636:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575350.166639:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575350.166641:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575350.166642:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575350.166644:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575350.166646:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166651:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575350.166653:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575375 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166662:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575350.166663:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575350.166664:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575350.166667:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575350.166669:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575350.166670:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306953913
00000100:00000001:2.0:1788575350.166672:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306954681
00000100:00000001:2.0:1788575350.166674:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306953913
00000100:00000001:2.0:1788575350.166675:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306954681
00000100:00000001:2.0:1788575350.166677:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575350.166678:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575350.166679:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575350.166681:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575350/real 1788575350]  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575350.166686:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575350.166687:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575350.166688:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575350.166690:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575350.166691:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575350.166692:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575350.166694:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575350.166699:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166703:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575350.166703:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166705:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575350.166705:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575350.166708:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166712:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575350.166715:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166719:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575350.166721:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166722:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575350.166722:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166724:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575350.166725:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575350.166726:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575350.166728:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575350.166729:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575350.166732:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575350.166736:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203072:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575350.166739:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166740:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575350.166742:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393202944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166746:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575350.166748:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575350.166749:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575350.166750:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575350.166751:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575350.166752:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575350.166754:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575350.166755:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:2.0:1788575350.166757:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575350.166758:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575350.166759:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:2.0:1788575350.166761:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575350.166762:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166763:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575350.166765:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393203072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575375 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575350.166769:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575350.166770:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575350.166771:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575350.166772:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575350.166773:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575350.166773:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575350.166775:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575350.166775:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575350.166777:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575350.166777:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575350.166778:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788575350.166780:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575350.166781:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166783:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575350.166786:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575350.166788:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575350.166789:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575350.166791:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575350.549464:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575350.549469:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575350.549473:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575350.549474:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575350.549477:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575350.549478:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575350.549538:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575350.549542:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575350.549545:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575350.677521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575350.677528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575350.677530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575350.677533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575350.677536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575350.677540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575350.677542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575350.677544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575350.677546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575351.125504:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306954681, at 4306954688
00000100:00000040:1.0:1788575351.125504:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306954681, at 4306954688
00000100:00000010:0.0:1788575351.125520:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000100:00000010:1.0:1788575351.125521:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00010000:00000001:3.0:1788575351.573500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575351.573507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575351.573513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575351.573515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575351.573519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575351.573520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575351.573585:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575351.573589:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575351.573593:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575351.701475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575351.701482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575351.701485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575351.701491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575351.701495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575351.701499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575351.701503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575351.701505:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575351.701509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575352.597489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575352.597495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575352.597499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575352.597500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575352.597503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575352.597504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575352.597547:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575352.597558:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575352.597562:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575352.725498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575352.725507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575352.725509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575352.725514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575352.725521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575352.725527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575352.725530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575352.725532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575352.725534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575353.621549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575353.621557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575353.621562:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575353.621563:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575353.621566:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575353.621567:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575353.621637:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575353.621641:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575353.621644:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575353.749551:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575353.749560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575353.749563:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575353.749571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575353.749575:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575353.749584:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575353.749588:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575353.749591:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575353.749595:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575354.645557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575354.645565:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575354.645570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575354.645574:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575354.645577:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575354.645581:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575354.645585:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575354.645588:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575354.645591:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575354.645594:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575354.645598:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575354.645600:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575354.645602:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575354.645604:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575354.645605:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575354.645607:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575354.645609:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575354.645611:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575354.645674:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575354.645678:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575354.645682:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575354.773551:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575354.773560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575354.773562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575354.773567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575354.773571:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575354.773576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575354.773580:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575354.773583:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575354.773587:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575355.285526:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575355.285538:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575355.285542:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575355.285545:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575355.285550:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575355.285558:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12286
00000400:00000200:0.0:1788575355.285571:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575355.285580:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575355.285584:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575355.285588:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575355.285592:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575355.285594:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575355.285597:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575355.285603:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575355.285611:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575355.285614:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575355.285617:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575355.285620:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575355.285622:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575355.285625:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575355.285627:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575355.285629:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575355.285631:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575355.285634:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575355.285637:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575355.285640:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575355.285643:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575355.285647:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575355.285651:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575355.285654:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575355.285657:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575355.285669:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575355.285674:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575355.285678:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575355.285680:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575355.285683:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575355.285687:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575355.285693:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575355.285702:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575355.285711:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575355.285724:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575355.285736:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575355.285741:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575355.285746:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575355.285771:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575355.285775:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575355.285779:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575355.285780:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575355.285782:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575355.285802:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575355.285805:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12286
00000400:00000200:0.0:1788575355.285810:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575355.285816:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575355.285819:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575355.285823:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575355.285825:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575355.285827:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575355.285830:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575355.285832:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788575355.285834:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575355.285839:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575355.285841:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788575355.285844:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575355.285845:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1788575355.285849:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575355.285854:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575355.285860:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575355.285863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575355.285868:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575355.285878:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575355.285888:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575355.285891:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575355.285894:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575355.285895:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575355.285901:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1788575355.285904:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575355.285905:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575355.285913:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575355.285918:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:2.0:1788575355.285925:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393203200:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575355.285926:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575355.285940:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575355.285940:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575355.285951:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575355.285954:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575355.285956:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575355.285956:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1788575355.285964:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575355.285968:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:2.0:1788575355.285969:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575355.285976:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1788575355.285977:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575355.285983:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575355.285991:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575355.285992:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1788575355.286001:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000400:00000010:2.0:1788575355.286003:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000020:00000040:0.0:1788575355.286007:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575355.286013:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1788575355.286014:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
02000000:00000001:0.0:1788575355.286016:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575355.286021:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575355.286023:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1788575355.286025:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393203200, portal 25
02000000:00000001:0.0:1788575355.286027:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.286031:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575355.286033:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
02000000:00000010:0.0:1788575355.286036:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000040:2.0:1788575355.286043:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575355.286046:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.286056:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:0.0:1788575355.286061:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788575355.286062:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575355.286067:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575355.286073:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393203200, offset 0
00000400:00000200:2.0:1788575355.286086:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575355.286087:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575355.286092:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575355.286096:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575355.286103:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575355.286108:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575355.286118:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575355.286128:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575355.286139:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788575355.286142:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575355.286142:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575355.286152:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330843:330843:256:4294967295] 192.168.202.110@tcp LPNI seq info [330800:330800:8:4294967295]
00000100:00080000:0.0:1788575355.286153:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575355.286171:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:2.0:1788575355.286171:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575355.286181:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575355.286185:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575355.286190:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788575355.286195:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:2.0:1788575355.286202:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788575355.286207:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12286
00000800:00000010:2.0:1788575355.286210:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000400:00000200:0.0:1788575355.286224:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:2.0:1788575355.286225:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575355.286240:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000400:00000200:0.0:1788575355.286244:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575355.286252:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:2.1:1788575355.286253:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575355.286261:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:0.0:1788575355.286262:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575355.286271:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575355.286277:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000100:2.0:1788575355.286279:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575355.286285:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000010:2.0:1788575355.286287:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000020:00000001:0.0:1788575355.286294:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575355.286298:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:2.0:1788575355.286300:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1788575355.286302:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575355.286308:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575355.286314:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575355.286318:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1788575355.286320:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575355.286329:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575355.286330:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000100:00000001:0.0:1788575355.286335:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575355.286337:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000040:0.0:1788575355.286344:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575355.286346:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:0.0:1788575355.286350:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:2.0:1788575355.286356:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575355.286360:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575355.286375:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575355.286378:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1788575355.286383:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575355.286386:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.286422:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.286424:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:0.0:1788575355.286435:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575355.286435:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575355.286445:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575355.286445:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575355.286453:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306959802
02000000:00000001:0.0:1788575355.286455:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575355.286460:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000010:2.0:1788575355.286460:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c420.
00000008:00100000:0.0:1788575355.286466:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575355.286470:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575355.286475:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.286475:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575355.286489:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306959802
00000100:00000010:0.0:1788575355.286490:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000100:00000010:2.0:1788575355.286495:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c9c0.
00000020:00000040:0.0:1788575355.286497:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:2.0:1788575355.286500:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:0.0:1788575355.286503:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575355.286506:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.286506:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575355.286521:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1788575355.286521:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575355.286530:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575355.286531:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
02000000:00000001:0.0:1788575355.286534:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575355.286541:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575355/real 1788575355]  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788575355.286543:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575355.286552:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1788575355.286558:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575355.286561:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00100000:0.0:1788575355.286562:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1788575355.286566:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575355.286575:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575355.286579:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:0.0:1788575355.286587:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575355.286588:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000001:0.0:1788575355.286592:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575355.286598:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575355.286607:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575355.286613:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575355.286675:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.286678:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:0.0:1788575355.286682:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575355.286683:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.286691:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:0.0:1788575355.286693:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575355.286696:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575355.286702:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12296)
00000100:00100000:2.0:1788575355.286710:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.286720:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575355.286725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.286731:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575355.286734:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.286735:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575355.286736:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.286738:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575355.286740:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575355.286743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575355.286746:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575355.286748:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575355.286751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575355.286757:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393203200:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575355.286760:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.286762:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575355.286764:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393203200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.286768:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575355.286770:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575355.286772:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575355.286775:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575355.286777:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575355.286778:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575355.286780:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575355.286781:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788575355.286797:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575355.286798:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575355.286800:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:2.0:1788575355.286803:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575355.286804:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.286806:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.286812:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575355.286813:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.286815:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575355.286816:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575355.286818:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575355.286821:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.286827:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575355.286828:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575355.286830:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575355.286831:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575355.286833:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575355.286835:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575355.286837:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575355.286839:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575355.286841:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575355.286842:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575355.286845:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575355.286846:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575355.286849:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575355.286851:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575355.286854:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575355.286856:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393203328, portal 10
00000100:00000001:2.0:1788575355.286858:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575355.286859:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00000040:2.0:1788575355.286863:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.286868:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575355.286870:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575355.286871:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575355.286873:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393203328, offset 0
00000400:00000200:2.0:1788575355.286876:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575355.286879:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575355.286882:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575355.286884:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575355.286886:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575355.286888:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575355.286889:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575355.286892:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330844:330844:256:4294967295] 192.168.202.110@tcp LPNI seq info [330801:330801:8:4294967295]
00000400:00000200:2.0:1788575355.286895:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575355.286900:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575355.286903:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575355.286905:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575355.286908:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575355.286910:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575355.286914:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575355.286916:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575355.286918:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575355.286920:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575355.286922:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575355.286924:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575355.286926:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575355.286928:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575355.286929:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575355.286932:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.286937:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575355.286939:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.286943:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.286944:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575355.286946:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575355.286949:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575355.286951:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575355.286953:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306958887
00000100:00000001:2.0:1788575355.286955:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306959802
00000100:00000001:2.0:1788575355.286956:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306958887
00000100:00000001:2.0:1788575355.286958:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306959802
00000100:00000001:2.0:1788575355.286959:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575355.286960:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.286962:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575355.286964:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575355/real 1788575355]  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575355.286970:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575355.286971:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575355.286973:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575355.286975:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575355.286977:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575355.286979:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575355.286983:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575355.286991:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.286999:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575355.287000:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287002:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575355.287004:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575355.287008:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287016:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575355.287021:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287028:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575355.287031:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287032:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575355.287033:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287034:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575355.287036:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575355.287038:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575355.287041:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575355.287043:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575355.287045:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575355.287051:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575355.287054:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575355.287057:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287061:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575355.287063:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575355.287064:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575355.287066:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575355.287067:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575355.287069:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575355.287071:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575355.287072:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575355.287074:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575355.287075:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575355.287078:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575355.287079:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575355.287082:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575355.287084:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575355.287087:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575355.287089:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393203456, portal 4
00000100:00000001:2.0:1788575355.287090:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575355.287091:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523505920 : -130484186045696 : ffff895348712300)
00000100:00000040:2.0:1788575355.287096:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287100:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575355.287102:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575355.287103:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575355.287105:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393203456, offset 0
00000400:00000200:2.0:1788575355.287108:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575355.287110:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575355.287113:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575355.287115:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575355.287117:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575355.287119:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575355.287120:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575355.287123:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330845:330845:256:4294967295] 192.168.202.110@tcp LPNI seq info [330802:330802:8:4294967295]
00000400:00000200:2.0:1788575355.287125:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575355.287130:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575355.287133:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575355.287135:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575355.287138:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575355.287140:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575355.287143:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575355.287144:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575355.287147:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575355.287148:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575355.287151:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575355.287153:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575355.287155:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575355.287156:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575355.287158:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575355.287160:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287166:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575355.287168:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287172:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.287173:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575355.287175:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575355.287178:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575355.287180:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575355.287181:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306959033
00000100:00000001:2.0:1788575355.287183:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306959802
00000100:00000001:2.0:1788575355.287185:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306959033
00000100:00000001:2.0:1788575355.287186:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306959802
00000100:00000001:2.0:1788575355.287188:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575355.287189:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.287190:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575355.287193:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575355/real 1788575355]  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575355.287197:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575355.287198:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575355.287199:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575355.287200:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575355.287201:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575355.287203:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575355.287205:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575355.287210:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287214:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575355.287215:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287216:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575355.287217:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575355.287219:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287223:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575355.287225:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287230:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575355.287232:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287233:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575355.287234:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287235:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575355.287236:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575355.287238:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575355.287242:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575355.287244:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575355.287248:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575355.287256:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575355.287260:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287262:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575355.287265:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393203328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287272:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575355.287274:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575355.287276:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575355.287279:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575355.287280:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575355.287282:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575355.287284:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575355.287286:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:2.0:1788575355.287288:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575355.287290:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575355.287291:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:2.0:1788575355.287294:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575355.287295:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287298:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575355.287301:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712300 x1875444393203456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575380 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575355.287309:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575355.287310:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575355.287312:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575355.287314:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575355.287315:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575355.287317:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575355.287319:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575355.287320:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:2.0:1788575355.287323:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575355.287324:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575355.287325:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712300.
00000100:00000001:2.0:1788575355.287328:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575355.287329:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287331:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575355.287335:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575355.287336:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575355.287337:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575355.287338:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575355.669534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575355.669543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575355.669548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575355.669549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575355.669552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575355.669554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575355.669636:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575355.669642:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575355.669646:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575355.797483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575355.797491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575355.797493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575355.797499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575355.797503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575355.797506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575355.797508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575355.797511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575355.797513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575356.245468:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306959802, at 4306959808
00000100:00000010:1.0:1788575356.245497:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000100:00000040:0.0:1788575356.245517:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306959802, at 4306959808
00000100:00000010:0.0:1788575356.245527:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00010000:00000001:3.0:1788575356.693520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575356.693527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575356.693535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575356.693537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575356.693540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575356.693541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575356.693598:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575356.693604:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575356.693607:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575356.821540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575356.821550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575356.821553:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575356.821559:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575356.821563:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575356.821567:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575356.821570:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575356.821572:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575356.821574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575357.717516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575357.717524:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575357.717526:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575357.717529:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575357.717533:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575357.717535:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575357.717536:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575357.717538:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575357.717541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575357.717542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575357.717545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575357.717546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575357.717616:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575357.717619:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575357.717622:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575357.845512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575357.845521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575357.845524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575357.845530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575357.845540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575357.845544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575357.845547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575357.845551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575357.845555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575358.741452:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575358.741457:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575358.741461:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575358.741462:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575358.741465:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575358.741466:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575358.741509:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575358.741512:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575358.741515:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575358.869508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575358.869516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575358.869528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575358.869538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575358.869542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575358.869546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575358.869549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575358.869552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575358.869555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575359.765484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575359.765490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575359.765494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575359.765495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575359.765497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575359.765498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575359.765560:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575359.765565:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575359.765568:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575359.893473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575359.893481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575359.893484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575359.893489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575359.893494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575359.893497:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575359.893500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575359.893503:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575359.893506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575360.405482:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575360.405495:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575360.405498:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575360.405502:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575360.405507:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575360.405514:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12291
00000400:00000200:1.0:1788575360.405523:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575360.405530:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575360.405534:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575360.405537:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575360.405541:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575360.405544:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575360.405547:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575360.405551:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575360.405553:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575360.405556:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575360.405560:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575360.405562:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575360.405565:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575360.405568:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575360.405570:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575360.405572:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575360.405574:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575360.405578:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575360.405581:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575360.405584:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575360.405587:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575360.405589:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575360.405591:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575360.405593:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575360.405595:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575360.405604:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788575360.405607:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575360.405610:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575360.405611:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575360.405614:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575360.405618:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575360.405622:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575360.405630:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788575360.405637:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575360.405647:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575360.405659:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575360.405663:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575360.405668:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575360.405693:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575360.405697:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575360.405700:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575360.405702:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575360.405704:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575360.405721:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575360.405725:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12291
00000400:00000200:1.0:1788575360.405730:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575360.405734:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575360.405737:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575360.405741:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788575360.405742:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575360.405747:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575360.405749:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1788575360.405751:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575360.405754:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1788575360.405756:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575360.405760:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575360.405764:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575360.405766:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575360.405766:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788575360.405771:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575360.405776:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575360.405781:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575360.405787:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1788575360.405789:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575360.405792:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575360.405793:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575360.405798:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575360.405800:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788575360.405802:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575360.405804:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1788575360.405807:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575360.405810:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575360.405813:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575360.405819:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575360.405820:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393203584:192.168.202.110@tcp:250:kworker.0
02000000:00000001:1.0:1788575360.405823:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575360.405829:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788575360.405831:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575360.405834:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788575360.405835:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575360.405839:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575360.405840:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575360.405845:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:1.0:1788575360.405847:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1788575360.405851:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
00000008:00000001:1.0:1788575360.405852:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575360.405857:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575360.405858:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575360.405864:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:1.0:1788575360.405866:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000400:00000010:2.0:1788575360.405869:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000020:00000040:1.0:1788575360.405870:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:2.0:1788575360.405875:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
02000000:00000001:1.0:1788575360.405876:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575360.405881:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575360.405884:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:2.0:1788575360.405885:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393203584, portal 25
02000000:00000001:1.0:1788575360.405888:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575360.405889:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575360.405892:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345478400 : -130483364073216 : ffff8953796f7100)
02000000:00000001:1.0:1788575360.405893:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575360.405898:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788575360.405901:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788575360.405908:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.405910:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575360.405914:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575360.405917:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00100000:1.0:1788575360.405919:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788575360.405920:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393203584, offset 0
00000400:00000200:2.0:1788575360.405932:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575360.405936:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575360.405940:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575360.405943:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788575360.405949:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575360.405952:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575360.405959:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575360.405965:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575360.405971:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575360.405976:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575360.405984:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330846:330846:256:4294967295] 192.168.202.110@tcp LPNI seq info [330803:330803:8:4294967295]
00000100:00000001:1.0:1788575360.405990:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575360.405990:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788575360.405996:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575360.406005:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:1.0:1788575360.406006:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575360.406011:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575360.406015:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575360.406019:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575360.406023:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12291
00000800:00000200:2.0:1788575360.406026:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788575360.406028:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:2.0:1788575360.406034:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000400:00000200:1.0:1788575360.406036:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575360.406041:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1788575360.406043:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788575360.406048:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.0:1788575360.406048:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788575360.406054:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788575360.406056:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788575360.406060:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575360.406061:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1788575360.406067:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000100:2.0:1788575360.406070:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:1.0:1788575360.406073:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000010:2.0:1788575360.406074:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000020:00000040:1.0:1788575360.406076:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575360.406079:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:2.0:1788575360.406080:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:1.0:1788575360.406082:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:2.0:1788575360.406087:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:1.0:1788575360.406088:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575360.406093:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575360.406093:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575360.406098:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000040:1.0:1788575360.406102:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575360.406103:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:1.0:1788575360.406107:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:2.0:1788575360.406110:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575360.406113:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575360.406120:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575360.406126:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788575360.406126:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575360.406132:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788575360.406133:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575360.406140:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.406144:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575360.406145:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575360.406147:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:1.0:1788575360.406151:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575360.406152:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575360.406156:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575360.406161:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575360.406162:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575360.406164:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306964921
00000008:00100000:1.0:1788575360.406167:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000010:2.0:1788575360.406169:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c0c0.
00000008:00000001:1.0:1788575360.406171:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575360.406175:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.406178:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000010:1.0:1788575360.406180:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000100:00000001:2.0:1788575360.406184:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306964921
00000020:00000040:1.0:1788575360.406185:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000010:2.0:1788575360.406190:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ca20.
02000000:00000001:1.0:1788575360.406191:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575360.406193:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:1.0:1788575360.406195:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.406197:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788575360.406200:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1788575360.406201:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575360.406205:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575360.406206:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
02000000:00000001:1.0:1788575360.406211:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575360.406212:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575360/real 1788575360]  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788575360.406220:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788575360.406227:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1788575360.406230:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575360.406232:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575360.406234:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000100:00100000:1.0:1788575360.406237:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575360.406238:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575360.406241:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575360.406244:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575360.406250:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575360.406260:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575360.406264:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575360.406265:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575360.406275:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406275:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575360.406278:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.406282:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575360.406285:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575360.406293:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575360.406296:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.406303:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788575360.406304:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575360.406308:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575360.406312:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575360.406318:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:1.0:1788575360.406322:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12301)
00000100:00000001:2.0:1788575360.406323:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.406328:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575360.406330:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.406333:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575360.406335:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575360.406339:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575360.406344:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575360.406348:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575360.406353:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575360.406364:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393203584:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575360.406369:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.406371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575360.406374:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7100 x1875444393203584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406381:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575360.406419:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575360.406422:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575360.406424:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575360.406426:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575360.406427:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575360.406429:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575360.406431:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:2.0:1788575360.406434:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575360.406435:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575360.406436:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7100.
00000100:00000001:2.0:1788575360.406439:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575360.406441:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.406443:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.406450:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575360.406452:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.406455:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575360.406457:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575360.406458:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575360.406463:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406471:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575360.406473:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575360.406475:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575360.406477:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575360.406479:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575360.406483:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203712:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575360.406487:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575360.406489:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575360.406490:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575360.406491:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575360.406494:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575360.406496:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575360.406500:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575360.406503:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575360.406507:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575360.406510:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393203712, portal 10
00000100:00000001:2.0:1788575360.406512:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575360.406514:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345481088 : -130483364070528 : ffff8953796f7b80)
00000100:00000040:2.0:1788575360.406521:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406528:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575360.406531:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575360.406533:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575360.406536:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393203712, offset 0
00000400:00000200:2.0:1788575360.406540:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575360.406545:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575360.406549:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575360.406553:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575360.406556:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575360.406558:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575360.406561:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575360.406564:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330847:330847:256:4294967295] 192.168.202.110@tcp LPNI seq info [330804:330804:8:4294967295]
00000400:00000200:2.0:1788575360.406569:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575360.406576:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575360.406581:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575360.406585:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575360.406590:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575360.406593:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575360.406598:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575360.406601:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575360.406606:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575360.406608:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575360.406611:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575360.406616:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575360.406619:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575360.406621:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575360.406624:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575360.406628:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406636:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575360.406640:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406647:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.406649:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575360.406652:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575360.406656:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575360.406659:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575360.406662:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306964007
00000100:00000001:2.0:1788575360.406665:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306964921
00000100:00000001:2.0:1788575360.406667:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306964007
00000100:00000001:2.0:1788575360.406669:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306964921
00000100:00000001:2.0:1788575360.406672:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575360.406674:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.406676:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575360.406680:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575360/real 1788575360]  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575360.406690:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575360.406691:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575360.406694:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575360.406696:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575360.406699:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575360.406701:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575360.406721:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575360.406731:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406739:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575360.406741:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.406742:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575360.406744:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575360.406748:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406755:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575360.406760:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406767:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575360.406770:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.406772:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575360.406773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.406775:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575360.406777:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575360.406781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575360.406784:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575360.406787:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575360.406791:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575360.406800:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203712:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575360.406805:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575360.406809:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406816:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575360.406817:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575360.406820:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575360.406822:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575360.406824:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575360.406827:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203840:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575360.406831:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575360.406833:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575360.406835:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575360.406836:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575360.406840:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575360.406843:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575360.406847:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575360.406849:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575360.406853:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575360.406856:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393203840, portal 4
00000100:00000001:2.0:1788575360.406858:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575360.406859:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345469440 : -130483364082176 : ffff8953796f4e00)
00000100:00000040:2.0:1788575360.406866:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406874:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575360.406876:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575360.406879:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575360.406881:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393203840, offset 0
00000400:00000200:2.0:1788575360.406885:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575360.406890:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575360.406894:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575360.406897:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575360.406900:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575360.406903:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575360.406905:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575360.406908:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330848:330848:256:4294967295] 192.168.202.110@tcp LPNI seq info [330805:330805:8:4294967295]
00000400:00000200:2.0:1788575360.406913:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575360.406920:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575360.406925:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575360.406928:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575360.406933:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575360.406936:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575360.406941:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575360.406944:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575360.406948:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575360.406950:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575360.406953:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575360.406956:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575360.406960:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575360.406962:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575360.406965:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575360.406969:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406978:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575360.406981:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.406988:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.406990:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575360.406994:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575360.406997:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575360.407002:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575360.407004:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306964153
00000100:00000001:2.0:1788575360.407006:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306964921
00000100:00000001:2.0:1788575360.407009:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306964153
00000100:00000001:2.0:1788575360.407011:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306964921
00000100:00000001:2.0:1788575360.407014:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575360.407016:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.407018:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575360.407023:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575360/real 1788575360]  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575360.407032:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575360.407033:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575360.407035:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575360.407038:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575360.407040:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575360.407042:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575360.407046:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575360.407055:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.407063:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575360.407064:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.407067:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575360.407068:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575360.407073:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.407080:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575360.407084:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.407092:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575360.407095:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.407097:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575360.407098:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.407100:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575360.407103:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575360.407106:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575360.407109:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575360.407112:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575360.407117:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575360.407126:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393203840:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575360.407131:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.407133:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575360.407137:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7b80 x1875444393203712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.407145:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575360.407147:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575360.407149:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575360.407152:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575360.407154:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575360.407155:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575360.407158:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575360.407160:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:2.0:1788575360.407163:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575360.407164:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575360.407166:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7b80.
00000100:00000001:2.0:1788575360.407169:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575360.407170:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.407172:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575360.407176:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4e00 x1875444393203840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575360.407183:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575360.407185:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575360.407186:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575360.407189:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575360.407191:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575360.407192:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575360.407195:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575360.407196:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:2.0:1788575360.407199:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575360.407200:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575360.407201:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4e00.
00000100:00000001:2.0:1788575360.407204:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575360.407205:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.407207:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575360.407211:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575360.407213:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575360.407214:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575360.407216:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575360.789467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575360.789473:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575360.789478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575360.789480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575360.789483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575360.789485:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575360.789548:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575360.789554:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575360.789558:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575360.917474:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575360.917483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575360.917485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575360.917491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575360.917495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575360.917499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575360.917502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575360.917505:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575360.917508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575361.365492:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306964921, at 4306964928
00000100:00000040:1.0:1788575361.365492:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306964921, at 4306964928
00000100:00000010:1.0:1788575361.365507:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000100:00000010:0.0:1788575361.365510:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00010000:00000001:3.0:1788575361.813475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575361.813482:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575361.813488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575361.813489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575361.813493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575361.813495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575361.813540:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575361.813544:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575361.813547:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575361.941469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575361.941476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575361.941478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575361.941482:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575361.941485:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575361.941488:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575361.941490:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575361.941493:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575361.941495:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575362.837467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575362.837475:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575362.837479:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575362.837480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575362.837483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575362.837485:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575362.837580:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575362.837587:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575362.837593:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575362.965555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575362.965566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575362.965569:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575362.965576:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575362.965581:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575362.965585:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575362.965589:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575362.965592:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575362.965596:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575363.861482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575363.861488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575363.861492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575363.861494:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575363.861496:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575363.861499:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575363.861502:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575363.861503:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575363.861505:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575363.861507:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575363.861509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575363.861511:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575363.861512:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575363.861513:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575363.861514:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575363.861516:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575363.861517:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575363.861519:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575363.861564:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575363.861568:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575363.861571:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575363.989463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575363.989469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575363.989471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575363.989475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575363.989478:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575363.989481:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575363.989483:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575363.989486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575363.989488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575364.885518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575364.885525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575364.885531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575364.885532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575364.885536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575364.885538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575364.885619:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575364.885625:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575364.885629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575365.013497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575365.013504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575365.013507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575365.013512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575365.013516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575365.013520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575365.013525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575365.013528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575365.013532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575365.525588:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575365.525603:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575365.525607:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575365.525613:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575365.525619:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575365.525645:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12296
00000400:00000200:2.0:1788575365.525658:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575365.525667:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575365.525671:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575365.525674:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575365.525679:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575365.525682:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575365.525685:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575365.525691:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575365.525694:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575365.525698:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575365.525701:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575365.525704:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575365.525707:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575365.525711:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575365.525713:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575365.525716:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575365.525718:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575365.525722:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575365.525726:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575365.525729:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575365.525733:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575365.525737:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575365.525741:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575365.525744:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575365.525748:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575365.525762:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00700.
00000020:00000040:2.0:1788575365.525767:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575365.525772:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575365.525774:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575365.525778:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575365.525781:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575365.525787:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575365.525797:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788575365.525805:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575365.525818:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575365.525831:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575365.525837:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575365.525843:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e69941e] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575365.525870:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575365.525874:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575365.525879:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575365.525880:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575365.525882:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575365.525885:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575365.525888:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12296
00000400:00000200:2.0:1788575365.525893:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575365.525896:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575365.525899:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575365.525904:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575365.525907:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575365.525910:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575365.525913:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575365.525915:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575365.525918:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575365.525918:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575365.525928:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1788575365.525929:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575365.525933:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1788575365.525936:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575365.525941:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575365.525946:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575365.525948:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575365.525964:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575365.525973:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575365.525976:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575365.525979:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575365.525981:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575365.525984:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1788575365.525987:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575365.525989:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575365.525999:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575365.526000:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00100000:3.0:1788575365.526005:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393203968:192.168.202.110@tcp:250:kworker.0
00000100:00080000:2.0:1788575365.526012:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575365.526014:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575365.526019:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575365.526024:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575365.526027:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575365.526028:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1788575365.526033:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:3.0:1788575365.526038:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575365.526041:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575365.526042:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575365.526054:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575365.526062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575365.526064:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1788575365.526066:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000008:00100000:2.0:1788575365.526072:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1788575365.526074:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000008:00000001:2.0:1788575365.526083:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575365.526084:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393203968, portal 25
00000100:00000001:2.0:1788575365.526091:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575365.526092:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575365.526096:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000010:2.0:1788575365.526103:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02680.
00000100:00000040:3.0:1788575365.526104:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788575365.526113:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:3.0:1788575365.526117:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1788575365.526120:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:3.0:1788575365.526122:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788575365.526126:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575365.526127:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000001:2.0:1788575365.526135:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:3.0:1788575365.526136:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393203968, offset 0
02000000:00000001:2.0:1788575365.526142:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575365.526147:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788575365.526149:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575365.526156:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1788575365.526164:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000400:00000200:3.0:1788575365.526168:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575365.526176:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575365.526180:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575365.526185:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1788575365.526186:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575365.526194:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00100000:2.0:1788575365.526201:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575365.526205:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330849:330849:256:4294967295] 192.168.202.110@tcp LPNI seq info [330806:330806:8:4294967295]
00000400:00000200:3.0:1788575365.526230:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575365.526240:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575365.526243:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575365.526253:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1788575365.526259:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000450d1599] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788575365.526269:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575365.526281:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00000001:2.0:1788575365.526295:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575365.526295:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575365.526303:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00080000:2.0:1788575365.526304:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575365.526315:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:3.1:1788575365.526316:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575365.526320:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:3.1:1788575365.526322:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00080000:2.0:1788575365.526324:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575365.526332:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000100:3.0:1788575365.526337:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575365.526340:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12296
00000800:00000010:3.0:1788575365.526343:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:2.0:1788575365.526350:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575365.526354:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575365.526360:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575365.526365:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575365.526370:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1788575365.526375:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575365.526380:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575365.526382:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000100:00000040:2.0:1788575365.526387:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:3.0:1788575365.526390:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:2.0:1788575365.526430:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575365.526434:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000020:00000001:2.0:1788575365.526438:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575365.526442:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000200:3.0:1788575365.526444:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575365.526450:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575365.526456:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575365.526467:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:3.0:1788575365.526468:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575365.526476:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575365.526478:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575365.526494:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575365.526494:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575365.526506:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575365.526506:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:2.0:1788575365.526520:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575365.526522:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575365.526526:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575365.526535:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575365.526537:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:3.0:1788575365.526539:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306970042
00000100:00080000:2.0:1788575365.526548:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000010:3.0:1788575365.526550:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:2.0:1788575365.526554:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575365.526557:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575365.526558:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:2.0:1788575365.526563:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575365.526566:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306970042
02000000:00000001:2.0:1788575365.526570:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575365.526571:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:2.0:1788575365.526579:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575365.526580:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000008:00100000:2.0:1788575365.526587:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:3.0:1788575365.526587:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575365.526601:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788575365.526602:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575365.526615:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1788575365.526616:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575365.526640:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02a00.
00000100:00100000:3.0:1788575365.526640:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575365/real 1788575365]  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788575365.526650:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575365.526658:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575365.526661:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575365.526669:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575365.526674:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575365.526679:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575365.526687:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000400:00000001:3.0:1788575365.526687:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575365.526693:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:2.0:1788575365.526695:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575365.526696:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575365.526704:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000100:00100000:2.0:1788575365.526707:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575365.526708:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575365.526717:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000001:2.0:1788575365.526727:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1788575365.526728:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575365.526732:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575365.526746:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cce910e4] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575365.526755:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575365.526787:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575365.526790:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:2.0:1788575365.526792:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575365.526793:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.526801:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:2.0:1788575365.526803:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575365.526804:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575365.526816:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12306)
00000100:00100000:3.0:1788575365.526817:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.526828:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575365.526833:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.526838:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575365.526841:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.526843:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575365.526844:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.526846:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575365.526847:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575365.526850:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575365.526853:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575365.526854:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575365.526856:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575365.526862:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393203968:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575365.526866:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.526868:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575365.526869:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393203968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.526873:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575365.526875:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575365.526878:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575365.526880:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575365.526882:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575365.526883:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575365.526885:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575365.526887:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575365.526889:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575365.526890:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575365.526892:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575365.526895:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575365.526896:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.526897:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.526903:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575365.526904:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575365.526906:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575365.526907:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575365.526908:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575365.526911:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.526916:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575365.526917:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575365.526919:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575365.526921:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575365.526922:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575365.526924:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204096:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575365.526926:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575365.526929:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575365.526930:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575365.526931:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575365.526933:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575365.526934:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575365.526937:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575365.526939:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575365.526941:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575365.526944:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393204096, portal 10
00000100:00000001:3.0:1788575365.526945:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575365.526946:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:3.0:1788575365.526950:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.526955:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575365.526956:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575365.526958:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575365.526959:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393204096, offset 0
00000400:00000200:3.0:1788575365.526962:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575365.526965:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575365.526967:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575365.526970:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575365.526972:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575365.526973:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575365.526975:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575365.526977:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330850:330850:256:4294967295] 192.168.202.110@tcp LPNI seq info [330807:330807:8:4294967295]
00000400:00000200:3.0:1788575365.526980:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575365.526985:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575365.526988:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575365.526990:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575365.526993:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575365.526995:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788575365.526999:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575365.527001:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788575365.527004:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575365.527005:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575365.527007:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575365.527010:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575365.527012:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575365.527013:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575365.527015:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575365.527017:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527022:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575365.527024:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527028:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575365.527029:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575365.527031:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575365.527034:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575365.527036:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575365.527037:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306969127
00000100:00000001:3.0:1788575365.527039:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306970042
00000100:00000001:3.0:1788575365.527041:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306969127
00000100:00000001:3.0:1788575365.527042:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306970042
00000100:00000001:3.0:1788575365.527044:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575365.527045:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575365.527046:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575365.527048:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575365/real 1788575365]  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575365.527053:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575365.527054:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575365.527055:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575365.527057:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575365.527058:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575365.527059:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575365.527061:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575365.527066:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527070:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575365.527071:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575365.527073:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575365.527075:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527079:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575365.527081:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527086:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575365.527088:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527089:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575365.527090:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527091:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575365.527092:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575365.527094:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575365.527096:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575365.527097:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575365.527100:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575365.527104:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204096:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575365.527107:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575365.527110:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527114:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575365.527115:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575365.527117:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575365.527118:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575365.527119:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575365.527121:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575365.527123:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575365.527124:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575365.527125:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575365.527126:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575365.527128:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575365.527130:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575365.527132:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575365.527134:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575365.527136:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575365.527138:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393204224, portal 4
00000100:00000001:3.0:1788575365.527139:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575365.527140:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:3.0:1788575365.527144:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527148:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575365.527149:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575365.527151:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575365.527152:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393204224, offset 0
00000400:00000200:3.0:1788575365.527155:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575365.527156:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575365.527158:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575365.527160:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575365.527163:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575365.527165:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575365.527167:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575365.527170:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330851:330851:256:4294967295] 192.168.202.110@tcp LPNI seq info [330808:330808:8:4294967295]
00000400:00000200:3.0:1788575365.527175:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575365.527181:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575365.527186:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575365.527190:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575365.527194:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575365.527197:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788575365.527201:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575365.527204:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788575365.527209:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575365.527211:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575365.527214:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575365.527217:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575365.527220:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575365.527222:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575365.527223:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575365.527226:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527237:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575365.527239:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527243:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575365.527245:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575365.527247:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575365.527249:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575365.527252:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575365.527253:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306969273
00000100:00000001:3.0:1788575365.527255:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306970042
00000100:00000001:3.0:1788575365.527257:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306969273
00000100:00000001:3.0:1788575365.527259:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306970042
00000100:00000001:3.0:1788575365.527260:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575365.527262:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575365.527263:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575365.527266:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575365/real 1788575365]  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575365.527270:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575365.527271:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575365.527272:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575365.527274:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575365.527275:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575365.527276:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575365.527278:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575365.527283:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527287:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575365.527288:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527289:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575365.527290:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575365.527292:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527296:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575365.527298:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527302:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575365.527304:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527305:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575365.527306:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527307:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575365.527309:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575365.527310:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575365.527314:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575365.527316:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575365.527321:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575365.527329:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575365.527333:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527335:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575365.527338:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393204096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527345:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575365.527347:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575365.527349:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575365.527352:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575365.527353:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575365.527355:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575365.527357:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575365.527359:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575365.527361:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575365.527363:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575365.527365:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575365.527367:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575365.527369:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575365.527375:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393204224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575365.527381:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575365.527383:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575365.527384:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575365.527387:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575365.527389:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575365.527390:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575365.527413:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575365.527415:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575365.527418:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575365.527419:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575365.527421:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:3.0:1788575365.527422:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575365.527423:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527425:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575365.527427:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575365.527428:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575365.527430:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575365.527431:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575365.909494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575365.909503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575365.909508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575365.909509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575365.909512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575365.909513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575365.909588:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575365.909592:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575365.909596:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575366.037539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575366.037553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575366.037555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575366.037562:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575366.037565:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575366.037569:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575366.037572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575366.037574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575366.037577:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575366.485481:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306970042, at 4306970048
00000100:00000040:1.0:1788575366.485481:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306970042, at 4306970048
00000100:00000010:1.0:1788575366.485500:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:0.0:1788575366.485506:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000800:00000200:1.0:1788575366.549545:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575366.549570:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788575366.549578:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575366.549787:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575366.549809:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575366.549810:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575366.549819:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575366.549823:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575366.933518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575366.933529:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575366.933532:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575366.933536:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575366.933542:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575366.933545:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575366.933548:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575366.933551:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575366.933556:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575366.933558:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575366.933563:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575366.933565:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575366.933631:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575366.933635:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575366.933638:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575367.061487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575367.061499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575367.061501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575367.061508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575367.061511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575367.061516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575367.061518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575367.061521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575367.061531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575367.957512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575367.957523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575367.957531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575367.957533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575367.957537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575367.957539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575367.957619:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575367.957623:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575367.957626:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575368.085539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575368.085551:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575368.085554:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575368.085562:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575368.085567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575368.085572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575368.085576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575368.085579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575368.085584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575368.981491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575368.981497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575368.981501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575368.981502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575368.981505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575368.981506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575368.981615:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575368.981620:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575368.981625:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575369.109499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575369.109505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575369.109507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575369.109511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575369.109513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575369.109518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575369.109521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575369.109524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575369.109527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575370.005459:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575370.005463:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575370.005467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575370.005468:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575370.005471:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575370.005472:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575370.005557:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575370.005594:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575370.005600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575370.133529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575370.133535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575370.133537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575370.133542:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575370.133545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575370.133547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575370.133550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575370.133552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575370.133554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575370.645563:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575370.645574:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575370.645576:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575370.645581:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575370.645585:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575370.645593:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12301
00000400:00000200:3.0:1788575370.645603:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575370.645609:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575370.645612:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575370.645615:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645618:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575370.645620:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645622:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575370.645626:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575370.645627:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575370.645630:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575370.645633:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575370.645634:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575370.645636:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575370.645638:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645640:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575370.645642:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645643:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575370.645646:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575370.645648:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645650:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575370.645653:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575370.645656:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575370.645659:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575370.645660:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575370.645662:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575370.645674:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000020:00000040:3.0:1788575370.645677:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575370.645680:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575370.645682:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575370.645684:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575370.645687:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575370.645691:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575370.645698:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575370.645706:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575370.645716:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575370.645724:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575370.645728:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575370.645732:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575370.645753:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575370.645756:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575370.645760:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575370.645761:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575370.645762:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575370.645764:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575370.645767:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12301
00000400:00000200:3.0:1788575370.645770:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575370.645772:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575370.645774:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575370.645777:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645779:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575370.645781:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645782:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575370.645784:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575370.645785:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575370.645786:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575370.645788:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575370.645789:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575370.645790:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575370.645793:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575370.645794:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575370.645795:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575370.645798:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645801:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575370.645802:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1788575370.645803:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575370.645807:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575370.645808:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575370.645809:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575370.645813:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575370.645820:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575370.645822:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575370.645823:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575370.645825:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575370.645831:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575370.645837:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575370.645841:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.645842:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000010:3.0:1788575370.645844:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575370.645847:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575370.645848:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575370.645850:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575370.645853:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575370.645855:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575370.645856:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575370.645856:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575370.645861:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575370.645862:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:3.0:1788575370.645864:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575370.645869:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575370.645870:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575370.645878:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393204352:192.168.202.110@tcp:250:kworker.0
00000100:00100000:3.0:1788575370.645879:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575370.645891:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575370.645893:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575370.645896:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575370.645899:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:3.0:1788575370.645899:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575370.645904:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575370.645910:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575370.645912:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:3.0:1788575370.645914:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575370.645917:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575370.645918:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575370.645919:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575370.645921:0:188751:0:(import.c:520:import_select_connection()) Process entered
02000000:00000010:2.0:1788575370.645923:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
00000100:00080000:3.0:1788575370.645923:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12301
02000000:00000001:2.0:1788575370.645930:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575370.645930:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575370.645935:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575370.645937:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575370.645939:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645940:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575370.645944:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575370.645944:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645946:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:2.0:1788575370.645953:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000020:00000001:3.0:1788575370.645957:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575370.645959:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575370.645961:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575370.645962:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575370.645964:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:2.0:1788575370.645965:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000020:00000001:3.0:1788575370.645968:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575370.645972:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645973:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575370.645975:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575370.645977:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393204352, portal 25
00000100:00000001:3.0:1788575370.645977:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575370.645981:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788575370.645985:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1788575370.645985:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575370.645989:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.645990:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
02000000:00000001:3.0:1788575370.645991:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575370.645994:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575370.645997:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575370.645999:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575370.646000:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575370.646003:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575370.646004:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575370.646005:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575370.646009:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788575370.646012:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575370.646017:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575370.646020:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575370.646024:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575370.646027:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575370.646029:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1788575370.646030:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575370.646036:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000010:3.0:1788575370.646036:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575370.646042:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575370.646044:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00100000:3.0:1788575370.646046:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788575370.646050:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393204352, offset 0
00000100:00000001:3.0:1788575370.646055:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575370.646058:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575370.646062:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575370.646066:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575370.646075:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575370.646077:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575370.646082:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575370.646093:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1788575370.646093:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575370.646104:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1788575370.646105:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12311)
00000400:00000200:2.0:1788575370.646112:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575370.646118:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575370.646121:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575370.646125:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330852:330852:256:4294967295] 192.168.202.110@tcp LPNI seq info [330809:330809:8:4294967295]
00000400:00000200:2.0:1788575370.646140:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575370.646152:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575370.646158:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575370.646164:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575370.646170:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575370.646174:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575370.646179:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575370.646182:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575370.646186:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575370.646188:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575370.646195:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575370.646199:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575370.646204:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575370.646206:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575370.646211:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575370.646215:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646229:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575370.646232:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646238:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.646239:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575370.646245:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575370.646248:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575370.646251:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306975161
00000100:00000010:2.0:1788575370.646256:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd80.
00000100:00000001:2.0:1788575370.646267:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575370.646275:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306975161
00000100:00000010:2.0:1788575370.646279:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c480.
00000100:00000001:2.0:1788575370.646283:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575370.646287:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575370.646290:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.646294:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575370.646299:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575370/real 1788575370]  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575370.646319:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575370.646320:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575370.646323:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575370.646326:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575370.646327:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575370.646330:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575370.646333:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575370.646339:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646344:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575370.646345:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646347:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575370.646348:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575370.646351:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646356:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575370.646360:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646365:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575370.646368:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575370.646370:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646371:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575370.646373:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575370.646375:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575370.646378:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575370.646380:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575370.646383:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575370.646389:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393204352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575370.646413:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646414:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575370.646417:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393204352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646422:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575370.646424:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575370.646427:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575370.646429:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575370.646431:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575370.646432:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575370.646434:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575370.646436:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575370.646440:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575370.646442:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575370.646443:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788575370.646446:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575370.646447:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646448:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646456:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575370.646457:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.646459:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575370.646460:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575370.646461:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575370.646464:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646469:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575370.646470:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575370.646472:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575370.646473:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575370.646475:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575370.646477:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575370.646479:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575370.646481:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575370.646482:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575370.646483:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575370.646486:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575370.646487:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575370.646490:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575370.646492:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575370.646495:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575370.646497:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393204480, portal 10
00000100:00000001:2.0:1788575370.646499:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575370.646500:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:2.0:1788575370.646505:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646510:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575370.646512:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575370.646513:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575370.646515:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393204480, offset 0
00000400:00000200:2.0:1788575370.646518:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575370.646521:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575370.646523:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575370.646525:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575370.646527:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575370.646529:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575370.646531:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575370.646533:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330853:330853:256:4294967295] 192.168.202.110@tcp LPNI seq info [330810:330810:8:4294967295]
00000400:00000200:2.0:1788575370.646536:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575370.646541:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575370.646544:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575370.646546:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575370.646548:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575370.646565:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575370.646568:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575370.646570:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575370.646573:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575370.646574:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575370.646577:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575370.646579:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575370.646582:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575370.646583:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575370.646585:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575370.646588:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646594:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575370.646596:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646600:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.646602:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575370.646604:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575370.646607:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575370.646610:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575370.646612:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306974247
00000100:00000001:2.0:1788575370.646614:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306975161
00000100:00000001:2.0:1788575370.646616:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306974247
00000100:00000001:2.0:1788575370.646618:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306975161
00000100:00000001:2.0:1788575370.646619:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575370.646620:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.646622:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575370.646624:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575370/real 1788575370]  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575370.646630:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575370.646631:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575370.646632:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575370.646634:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575370.646635:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575370.646637:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575370.646639:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575370.646644:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646648:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575370.646649:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646650:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575370.646651:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575370.646653:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646657:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575370.646659:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646663:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575370.646665:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646666:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575370.646667:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646668:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575370.646670:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575370.646672:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575370.646674:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575370.646675:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575370.646677:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575370.646682:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575370.646685:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575370.646688:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646692:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575370.646693:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575370.646694:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575370.646696:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575370.646697:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575370.646699:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204608:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575370.646700:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575370.646702:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575370.646703:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575370.646704:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575370.646707:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575370.646709:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575370.646711:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575370.646712:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575370.646715:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000100:00000200:2.0:1788575370.646717:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393204608, portal 4
00000100:00000001:2.0:1788575370.646718:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575370.646719:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788575370.646724:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646729:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575370.646731:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575370.646732:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575370.646734:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393204608, offset 0
00000400:00000200:2.0:1788575370.646736:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575370.646739:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575370.646741:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575370.646743:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575370.646745:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575370.646747:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575370.646749:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575370.646751:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330854:330854:256:4294967295] 192.168.202.110@tcp LPNI seq info [330811:330811:8:4294967295]
00000400:00000200:2.0:1788575370.646754:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575370.646758:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575370.646761:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575370.646762:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575370.646765:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575370.646767:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575370.646770:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575370.646772:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575370.646774:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575370.646776:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575370.646778:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575370.646781:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575370.646783:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575370.646784:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575370.646785:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575370.646788:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646793:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575370.646795:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646799:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.646800:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575370.646802:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575370.646804:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575370.646807:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575370.646808:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306974393
00000100:00000001:2.0:1788575370.646810:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306975161
00000100:00000001:2.0:1788575370.646811:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306974393
00000100:00000001:2.0:1788575370.646813:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306975161
00000100:00000001:2.0:1788575370.646815:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575370.646817:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.646818:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575370.646820:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575370/real 1788575370]  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575370.646825:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575370.646826:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575370.646827:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575370.646828:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:2.0:1788575370.646829:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:2.0:1788575370.646830:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575370.646832:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575370.646838:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646842:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575370.646843:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646844:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575370.646845:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575370.646847:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646851:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575370.646853:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646857:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575370.646858:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646859:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575370.646860:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646862:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575370.646863:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575370.646865:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575370.646867:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575370.646869:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575370.646871:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575370.646876:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204608:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575370.646878:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646879:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575370.646881:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393204480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646885:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575370.646886:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575370.646887:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575370.646888:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575370.646889:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575370.646890:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575370.646891:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575370.646892:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575370.646894:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575370.646895:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575370.646896:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575370.646898:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575370.646899:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646901:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575370.646903:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393204608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575370.646907:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575370.646908:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575370.646909:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575370.646910:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575370.646912:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575370.646912:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575370.646914:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575370.646915:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575370.646917:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575370.646917:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575370.646918:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575370.646920:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575370.646921:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646922:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575370.646924:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575370.646926:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575370.646927:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575370.646929:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575371.029537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575371.029560:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575371.029568:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575371.029569:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575371.029572:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575371.029573:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575371.029635:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575371.029638:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575371.029642:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575371.157461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575371.157468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575371.157470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575371.157474:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575371.157477:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575371.157480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575371.157483:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575371.157484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575371.157487:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575371.605456:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306975161, at 4306975168
00000100:00000040:1.0:1788575371.605456:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306975161, at 4306975168
00000100:00000010:1.0:1788575371.605472:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000100:00000010:0.0:1788575371.605476:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00010000:00000001:3.0:1788575372.053469:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575372.053474:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575372.053478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575372.053479:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575372.053482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575372.053483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575372.053586:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575372.053592:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575372.053597:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575372.181485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575372.181493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575372.181496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575372.181502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575372.181505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575372.181510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575372.181514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575372.181517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575372.181521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575373.077559:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575373.077566:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575373.077572:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575373.077574:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575373.077579:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575373.077580:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575373.077684:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575373.077691:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575373.077698:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575373.205447:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575373.205455:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575373.205458:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575373.205464:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575373.205468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575373.205472:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575373.205475:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575373.205477:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575373.205481:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575374.101524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575374.101529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575374.101533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575374.101541:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575374.101543:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575374.101546:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575374.101548:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575374.101550:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575374.101552:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575374.101553:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575374.101556:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575374.101557:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575374.101558:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575374.101559:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575374.101560:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575374.101561:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575374.101563:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575374.101564:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575374.101616:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575374.101619:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575374.101622:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575374.229541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575374.229548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575374.229549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575374.229554:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575374.229556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575374.229559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575374.229561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575374.229563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575374.229566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575375.125525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575375.125531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575375.125534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575375.125535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575375.125538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575375.125539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575375.125596:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575375.125599:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575375.125602:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575375.253504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575375.253513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575375.253515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575375.253521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575375.253525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575375.253529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575375.253532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575375.253535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575375.253538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575375.765525:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575375.765533:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575375.765535:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575375.765538:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575375.765541:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575375.765547:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12306
00000400:00000200:0.0:1788575375.765556:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575375.765561:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575375.765564:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575375.765566:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575375.765569:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575375.765570:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575375.765572:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575375.765576:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575375.765577:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575375.765579:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575375.765583:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575375.765584:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575375.765586:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575375.765588:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575375.765589:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575375.765591:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575375.765592:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575375.765594:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575375.765596:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575375.765598:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575375.765600:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575375.765602:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575375.765605:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575375.765606:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575375.765608:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575375.765616:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788575375.765619:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575375.765621:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575375.765622:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575375.765624:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575375.765627:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575375.765630:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575375.765636:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575375.765641:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575375.765648:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575375.765655:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575375.765658:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575375.765662:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575375.765682:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575375.765684:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575375.765687:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575375.765688:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575375.765690:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575375.765692:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575375.765694:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12306
00000400:00000200:0.0:1788575375.765696:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575375.765700:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575375.765703:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575375.765706:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575375.765709:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575375.765712:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575375.765714:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575375.765715:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575375.765719:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788575375.765720:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575375.765722:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1788575375.765724:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575375.765729:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575375.765730:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575375.765733:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575375.765739:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575375.765745:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575375.765746:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575375.765749:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575375.765753:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575375.765753:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575375.765757:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575375.765759:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575375.765760:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575375.765765:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00100000:2.0:1788575375.765766:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393204736:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575375.765773:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575375.765774:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1788575375.765778:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575375.765779:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575375.765783:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575375.765786:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575375.765787:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788575375.765792:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575375.765794:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788575375.765797:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575375.765798:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575375.765804:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575375.765805:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:2.0:1788575375.765808:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
02000000:00000001:0.0:1788575375.765810:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575375.765815:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:2.0:1788575375.765815:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65880.
00000008:00100000:0.0:1788575375.765819:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000200:2.0:1788575375.765822:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393204736, portal 25
00000008:00000001:0.0:1788575375.765825:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.765826:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575375.765830:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.765830:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000010:0.0:1788575375.765838:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000100:00000040:2.0:1788575375.765839:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575375.765846:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575375.765849:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1788575375.765853:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:2.0:1788575375.765854:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575375.765857:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575375.765859:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000001:0.0:1788575375.765862:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:2.0:1788575375.765864:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393204736, offset 0
02000000:00000001:0.0:1788575375.765867:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575375.765871:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575375.765872:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575375.765879:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000400:00000200:2.0:1788575375.765880:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575375.765886:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575375.765888:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575375.765894:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:0.0:1788575375.765896:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575375.765899:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575375.765906:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575375.765912:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788575375.765916:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575375.765919:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330855:330855:256:4294967295] 192.168.202.110@tcp LPNI seq info [330812:330812:8:4294967295]
00000100:00000001:0.0:1788575375.765922:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575375.765926:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575375.765931:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575375.765940:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575375.765954:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575375.765956:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575375.765961:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000100:00080000:0.0:1788575375.765962:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575375.765967:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575375.765970:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:2.0:1788575375.765972:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1788575375.765973:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575375.765980:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:2.1:1788575375.765983:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1788575375.765986:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12306
00000400:00000200:0.0:1788575375.765993:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575375.765997:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575375.765999:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575375.766002:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575375.766005:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575375.766006:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575375.766008:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575375.766013:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575375.766016:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1788575375.766018:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575375.766021:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306980281
00000020:00000040:0.0:1788575375.766022:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575375.766026:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000010:2.0:1788575375.766028:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c540.
00000020:00000040:0.0:1788575375.766030:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575375.766034:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788575375.766035:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:0.0:1788575375.766039:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788575375.766043:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306980281
00000100:00000040:0.0:1788575375.766046:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000010:2.0:1788575375.766048:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c900.
00000100:00000001:0.0:1788575375.766051:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575375.766053:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:0.0:1788575375.766058:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575375.766061:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575375.766062:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575375.766068:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575375.766069:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575375.766076:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575375.766076:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575375.766082:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766083:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1788575375.766085:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575375.766089:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575375.766092:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:0.0:1788575375.766093:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575375.766097:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575375.766097:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000008:00100000:0.0:1788575375.766101:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575375.766103:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00000001:0.0:1788575375.766106:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766107:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575375.766110:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766111:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000010:0.0:1788575375.766119:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000100:00000040:2.0:1788575375.766119:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575375.766126:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:2.0:1788575375.766129:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788575375.766132:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575375.766132:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575375.766136:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575375.766136:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575375.766140:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575375.766141:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575375.766143:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575375.766146:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575375.766149:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575375.766154:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00100000:2.0:1788575375.766154:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204864:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575375.766161:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766162:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575375.766166:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575375.766169:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575375.766171:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575375.766173:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575375.766179:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575375.766184:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575375.766195:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575375.766197:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1788575375.766200:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f9f8.
00000400:00000010:2.0:1788575375.766205:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65c38.
00000100:00000200:2.0:1788575375.766208:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393204864, portal 10
00000100:00000001:0.0:1788575375.766210:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766212:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575375.766215:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523500544 : -130484186051072 : ffff895348710e00)
00000100:00000040:0.0:1788575375.766221:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575375.766225:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575375.766248:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788575375.766250:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575375.766251:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575375.766254:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000040:0.0:1788575375.766257:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1788575375.766258:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393204864, offset 0
00000400:00000200:2.0:1788575375.766265:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788575375.766266:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575375.766270:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575375.766275:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12317)
00000400:00000200:2.0:1788575375.766278:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575375.766283:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575375.766286:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575375.766289:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575375.766293:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575375.766296:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330856:330856:255:4294967295] 192.168.202.110@tcp LPNI seq info [330813:330813:7:4294967295]
00000400:00000200:2.0:1788575375.766301:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575375.766309:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575375.766313:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575375.766318:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133a00.
00000800:00000200:2.0:1788575375.766322:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575375.766326:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575375.766329:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575375.766333:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575375.766336:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575375.766338:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306979366
00000100:00000001:2.0:1788575375.766341:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306980281
00000100:00000001:2.0:1788575375.766343:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306979366
00000100:00000001:2.0:1788575375.766346:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306980281
00000100:00000001:2.0:1788575375.766348:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575375.766350:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766353:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766355:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575375.766359:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575375.766361:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575375.766363:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575375.766364:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575375.766366:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575375.766370:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575375.766378:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575375.766379:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575375.766381:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575375.766383:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575375.766385:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575375.766388:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204992:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575375.766437:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575375.766440:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575375.766442:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575375.766444:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575375.766448:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575375.766451:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575375.766455:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575375.766458:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fa50.
00000400:00000010:2.0:1788575375.766463:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c653b8.
00000100:00000200:2.0:1788575375.766466:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393204992, portal 4
00000100:00000001:2.0:1788575375.766468:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575375.766470:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523505920 : -130484186045696 : ffff895348712300)
00000100:00000040:2.0:1788575375.766493:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575375.766501:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575375.766503:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575375.766506:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575375.766509:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393204992, offset 0
00000400:00000200:2.0:1788575375.766514:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575375.766519:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575375.766523:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575375.766526:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575375.766528:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575375.766538:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575375.766541:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575375.766544:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330857:330857:254:4294967295] 192.168.202.110@tcp LPNI seq info [330814:330814:6:4294967295]
00000400:00000200:2.0:1788575375.766548:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575375.766554:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575375.766560:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575375.766564:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575375.766567:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575375.766572:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575375.766574:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575375.766581:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575375.766585:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575375.766587:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306979513
00000100:00000001:2.0:1788575375.766590:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306980281
00000100:00000001:2.0:1788575375.766592:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306979513
00000100:00000001:2.0:1788575375.766594:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306980281
00000100:00000001:2.0:1788575375.766596:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575375.766599:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766602:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766604:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575375.766608:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575375.766609:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575375.766611:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575375.766612:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575375.766614:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575375.766616:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575376.149530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575376.149535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575376.149539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575376.149540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575376.149543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575376.149544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575376.149606:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575376.149611:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575376.149615:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575376.277517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575376.277526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575376.277527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575376.277532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575376.277535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575376.277538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575376.277540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575376.277543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575376.277545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575376.725533:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306980281, at 4306980288
00000100:00000040:1.0:1788575376.725533:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306980281, at 4306980288
00000100:00000010:1.0:1788575376.725547:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000100:00000010:0.0:1788575376.725548:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00010000:00000001:3.0:1788575377.173518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575377.173525:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575377.173527:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575377.173530:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575377.173534:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575377.173536:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575377.173537:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575377.173539:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575377.173541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575377.173542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575377.173545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575377.173545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575377.173592:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575377.173596:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575377.173598:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575377.301513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575377.301519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575377.301521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575377.301525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575377.301528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575377.301531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575377.301533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575377.301535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575377.301537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575378.197512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575378.197518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575378.197524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575378.197525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575378.197528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575378.197529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575378.197561:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575378.197565:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575378.197568:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575378.325546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575378.325554:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575378.325556:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575378.325562:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575378.325566:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575378.325570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575378.325574:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575378.325576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575378.325580:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1788575378.837747:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:2.0:1788575378.837792:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1788575378.837801:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575378.837806:0:2349:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:2.0:1788575378.837811:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575378.837814:0:2349:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1788575378.837820:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1788575378.837825:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:2.0:1788575378.837830:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1788575378.837835:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1788575378.837843:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575378.837846:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575378.837851:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575378.837861:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575378.837898:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575378.837904:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575378.837911:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575378.837913:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:2.0:1788575378.837925:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1788575378.837927:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133a00.
00000400:00000200:2.0:1788575378.837931:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1788575378.837934:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1788575378.837938:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575378.837940:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575378.837943:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575378.837949:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575378.837956:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575378.837958:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575378.837965:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575378.837966:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:2.0:1788575378.837972:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1788575378.837974:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575378.837977:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1788575378.837981:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1788575378.837983:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575378.837986:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575378.837988:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575378.837993:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575378.838001:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575378.838004:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575378.838010:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575378.838012:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575378.839209:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575378.839216:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575378.839220:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575378.839234:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575375/real 1788575378]  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575378.839265:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575378.839268:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575378.839272:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fa50.
00000400:00000200:3.0:1788575378.839277:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c653b8
00000400:00000010:3.0:1788575378.839279:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c653b8.
00000100:00000001:3.0:1788575378.839284:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575378.839288:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575378.839299:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839310:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575378.839312:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839315:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575378.839318:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575378.839322:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839329:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575378.839334:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839341:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575378.839345:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839346:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575378.839347:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839350:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575378.839352:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575378.839356:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575378.839361:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575378.839364:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575378.839369:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575378.839377:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204992:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575378.839382:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575378.839386:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575375/real 1788575378]  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575378.839421:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575378.839423:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575378.839439:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f9f8.
00000400:00000200:3.0:1788575378.839444:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65c38
00000400:00000010:3.0:1788575378.839448:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65c38.
00000100:00000001:3.0:1788575378.839451:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575378.839455:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575378.839465:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839473:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575378.839474:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839476:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575378.839478:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575378.839482:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839488:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575378.839492:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839499:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575378.839501:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839547:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575378.839549:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839551:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575378.839553:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575378.839556:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575378.839561:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575378.839563:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575378.839568:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575378.839576:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393204864:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575378.839581:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575378.839585:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575375/real 1788575378]  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575378.839593:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575378.839595:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575378.839597:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:3.0:1788575378.839600:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65880
00000400:00000010:3.0:1788575378.839602:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65880.
00000100:00000001:3.0:1788575378.839604:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575378.839607:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575378.839615:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839621:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575378.839623:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839625:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575378.839626:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575378.839629:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839641:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575378.839645:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839652:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575378.839654:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839655:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575378.839656:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839658:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575378.839660:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575378.839662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575378.839665:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575378.839667:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575378.839671:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575378.839679:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393204736:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575378.839683:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839685:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575378.839699:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712300 x1875444393204992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839706:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575378.839709:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575378.839713:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575378.839717:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575378.839720:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575378.839722:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575378.839724:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575378.839727:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:3.0:1788575378.839731:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575378.839733:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575378.839735:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712300.
00000100:00000001:3.0:1788575378.839739:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575378.839741:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839743:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575378.839747:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710e00 x1875444393204864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839753:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575378.839755:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575378.839756:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575378.839759:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575378.839760:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575378.839761:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575378.839763:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575378.839765:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:3.0:1788575378.839767:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575378.839768:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575378.839770:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710e00.
00000100:00000001:3.0:1788575378.839772:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575378.839774:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839775:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575378.839779:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393204736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575378.839784:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575378.839785:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575378.839787:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575378.839790:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575378.839791:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575378.839792:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575378.839794:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575378.839796:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:3.0:1788575378.839798:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575378.839799:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575378.839800:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:3.0:1788575378.839803:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575378.839804:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839807:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575378.839815:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575378.839817:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575378.839820:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575378.839821:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575379.221533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575379.221540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575379.221546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575379.221548:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575379.221551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575379.221553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575379.221613:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575379.221616:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575379.221618:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575379.349489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575379.349495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575379.349496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575379.349501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575379.349505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575379.349508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575379.349510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575379.349513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575379.349515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575380.245476:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575380.245484:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575380.245490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575380.245491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575380.245494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575380.245496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575380.245539:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575380.245542:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575380.245545:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575380.373532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575380.373542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575380.373544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575380.373548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575380.373552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575380.373556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575380.373558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575380.373560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575380.373562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575380.885557:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575380.885565:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575380.885568:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575380.885571:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575380.885574:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575380.885580:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12312
00000400:00000200:1.0:1788575380.885587:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575380.885592:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575380.885595:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575380.885597:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885600:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575380.885602:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885603:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575380.885607:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575380.885608:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575380.885610:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575380.885613:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575380.885615:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575380.885616:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575380.885618:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885620:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575380.885621:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885622:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575380.885625:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575380.885627:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885628:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575380.885631:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575380.885633:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575380.885635:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575380.885637:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575380.885639:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575380.885644:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788575380.885646:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575380.885649:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575380.885651:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575380.885654:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575380.885657:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575380.885660:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575380.885668:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788575380.885675:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575380.885684:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575380.885695:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575380.885700:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575380.885704:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575380.885733:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575380.885737:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575380.885740:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575380.885742:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575380.885744:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575380.885747:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575380.885750:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12312
00000400:00000200:1.0:1788575380.885754:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575380.885756:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575380.885758:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575380.885760:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885761:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575380.885763:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885764:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575380.885766:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575380.885767:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575380.885768:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575380.885769:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575380.885771:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575380.885773:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575380.885775:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885777:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575380.885779:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885781:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575380.885783:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575380.885786:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885789:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575380.885790:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575380.885797:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575380.885799:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575380.885801:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575380.885806:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575380.885809:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575380.885811:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575380.885815:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788575380.885817:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575380.885820:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575380.885822:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575380.885824:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575380.885825:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575380.885827:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575380.885831:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788575380.885835:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575380.885840:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575380.885849:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575380.885851:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575380.885856:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575380.885869:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575380.885871:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575380.885875:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575380.885877:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575380.885878:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575380.885881:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575380.885884:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12312
00000400:00000200:1.0:1788575380.885889:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575380.885892:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575380.885894:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575380.885898:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885900:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575380.885902:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885904:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575380.885907:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575380.885908:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575380.885909:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575380.885911:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575380.885919:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575380.885921:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575380.885924:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885926:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575380.885928:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885930:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575380.885932:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575380.885935:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575380.885937:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575380.885940:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575380.885943:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575380.885945:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575380.885947:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575380.885950:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575380.885952:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575380.885954:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575380.885958:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788575380.885961:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575380.885963:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575380.885964:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575380.885966:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575380.885968:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575380.885969:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575380.885974:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788575380.885977:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575380.885982:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575380.885990:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575380.885992:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575380.885996:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575380.886004:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575380.886007:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575380.886012:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575380.886017:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12322)
00000100:00000001:3.0:1788575380.886149:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575380.886156:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575380.886159:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575380.886167:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.886181:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575380.886183:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575380.886187:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575380.886190:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575380.886193:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575380.886196:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393205120:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575380.886200:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575380.886205:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575380.886208:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575380.886211:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575380.886216:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575380.886219:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575380.886225:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575380.886231:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575380.886239:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575380.886247:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393205120, portal 25
00000100:00000001:3.0:1788575380.886250:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575380.886252:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345471232 : -130483364080384 : ffff8953796f5500)
00000100:00000040:3.0:1788575380.886259:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.886859:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575380.886863:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575380.886868:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575380.886873:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393205120, offset 0
00000400:00000200:3.0:1788575380.886881:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575380.886892:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575380.886900:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575380.886908:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575380.886912:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575380.886915:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575380.886918:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575380.886923:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330858:330858:256:4294967295] 192.168.202.110@tcp LPNI seq info [330815:330815:8:4294967295]
00000400:00000200:3.0:1788575380.886942:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575380.886953:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575380.886960:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575380.886966:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575380.886974:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575380.886978:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575380.886984:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575380.886988:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575380.886993:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575380.887004:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575380.887008:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575380.887015:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575380.887022:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575380.887025:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575380.887030:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575380.887036:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887047:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575380.887051:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887060:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575380.887061:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575380.887070:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575380.887074:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575380.887077:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306985402
00000100:00000010:3.0:1788575380.887083:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575380.887090:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575380.887098:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306985402
00000100:00000010:3.0:1788575380.887103:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575380.887107:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575380.887118:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575380.887121:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575380.887124:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575380.887129:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575380/real 1788575380]  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575380.887140:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575380.887142:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575380.887144:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575380.887148:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575380.887149:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575380.887152:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575380.887156:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575380.887164:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887171:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575380.887174:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887177:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575380.887179:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575380.887183:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887190:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575380.887195:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887203:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575380.887205:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887207:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575380.887208:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887210:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575380.887211:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575380.887214:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575380.887217:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575380.887219:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575380.887222:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575380.887227:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393205120:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575380.887231:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575380.887233:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887239:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575380.887240:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575380.887242:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575380.887243:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575380.887245:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575380.887247:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393205248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575380.887250:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575380.887252:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575380.887254:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575380.887256:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575380.887259:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575380.887260:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575380.887264:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575380.887266:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575380.887269:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575380.887271:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393205248, portal 10
00000100:00000001:3.0:1788575380.887273:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575380.887274:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:3.0:1788575380.887279:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887284:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575380.887286:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575380.887288:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575380.887289:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393205248, offset 0
00000400:00000200:3.0:1788575380.887292:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575380.887295:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575380.887298:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575380.887300:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575380.887302:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575380.887304:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575380.887305:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575380.887308:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330859:330859:256:4294967295] 192.168.202.110@tcp LPNI seq info [330816:330816:8:4294967295]
00000400:00000200:3.0:1788575380.887311:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575380.887316:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575380.887318:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575380.887320:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575380.887323:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575380.887325:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575380.887329:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575380.887332:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575380.887336:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575380.887338:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575380.887341:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575380.887344:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575380.887347:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575380.887349:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575380.887352:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575380.887355:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887363:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575380.887366:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887373:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575380.887375:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575380.887379:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575380.887383:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575380.887387:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575380.887389:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306984487
00000100:00000001:3.0:1788575380.887499:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306985402
00000100:00000001:3.0:1788575380.887502:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306984488
00000100:00000001:3.0:1788575380.887503:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306985402
00000100:00000001:3.0:1788575380.887505:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575380.887507:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575380.887508:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575380.887512:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575380/real 1788575380]  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575380.887518:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575380.887519:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575380.887521:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575380.887524:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575380.887525:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575380.887527:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575380.887529:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575380.887535:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887539:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575380.887540:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887541:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575380.887542:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575380.887544:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887549:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575380.887551:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887556:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575380.887558:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887559:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575380.887559:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887561:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575380.887562:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575380.887564:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575380.887566:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575380.887567:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575380.887569:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575380.887574:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393205248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575380.887577:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575380.887580:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887584:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575380.887585:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575380.887587:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575380.887588:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575380.887589:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575380.887591:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393205376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575380.887593:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575380.887594:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575380.887595:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575380.887597:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575380.887600:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575380.887601:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575380.887604:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575380.887606:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575380.887608:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575380.887610:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393205376, portal 4
00000100:00000001:3.0:1788575380.887611:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575380.887612:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:3.0:1788575380.887616:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887621:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575380.887622:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575380.887624:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575380.887625:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393205376, offset 0
00000400:00000200:3.0:1788575380.887628:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575380.887631:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575380.887633:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575380.887635:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575380.887637:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575380.887639:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575380.887641:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575380.887643:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330860:330860:256:4294967295] 192.168.202.110@tcp LPNI seq info [330817:330817:8:4294967295]
00000400:00000200:3.0:1788575380.887646:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575380.887650:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575380.887653:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575380.887655:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575380.887658:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575380.887660:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575380.887663:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575380.887665:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575380.887667:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575380.887669:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575380.887671:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575380.887673:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575380.887676:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575380.887677:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575380.887678:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575380.887681:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887686:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575380.887688:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887692:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575380.887694:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575380.887696:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575380.887698:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575380.887701:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575380.887702:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306984634
00000100:00000001:3.0:1788575380.887704:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306985402
00000100:00000001:3.0:1788575380.887706:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306984634
00000100:00000001:3.0:1788575380.887708:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306985402
00000100:00000001:3.0:1788575380.887709:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575380.887711:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575380.887712:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575380.887715:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575380/real 1788575380]  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575380.887719:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575380.887720:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575380.887721:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575380.887722:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575380.887723:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575380.887724:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575380.887726:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575380.887732:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887736:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575380.887737:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887738:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575380.887739:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575380.887741:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887745:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575380.887748:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887752:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575380.887753:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887754:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575380.887755:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887756:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575380.887758:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575380.887759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575380.887762:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575380.887763:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575380.887766:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575380.887770:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393205376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575380.887773:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887774:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575380.887776:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5500 x1875444393205120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887780:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575380.887782:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575380.887784:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575380.887786:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575380.887788:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575380.887789:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575380.887790:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575380.887792:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:3.0:1788575380.887794:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575380.887796:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575380.887797:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5500.
00000100:00000001:3.0:1788575380.887799:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575380.887800:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887801:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575380.887804:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393205248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887808:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575380.887809:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575380.887809:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575380.887811:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575380.887812:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575380.887813:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575380.887814:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575380.887815:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:3.0:1788575380.887817:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575380.887817:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575380.887818:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:3.0:1788575380.887819:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575380.887820:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887821:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575380.887823:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393205376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575380.887827:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575380.887827:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575380.887828:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575380.887829:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575380.887830:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575380.887831:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575380.887832:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575380.887833:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:3.0:1788575380.887834:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575380.887835:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575380.887835:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:3.0:1788575380.887837:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575380.887837:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887839:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575380.887845:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575380.887846:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575380.887848:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575380.887849:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575381.269523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575381.269529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575381.269535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575381.269536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575381.269540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575381.269541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575381.269599:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575381.269602:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575381.269605:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575381.397487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575381.397495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575381.397498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575381.397503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575381.397506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575381.397509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575381.397511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575381.397513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575381.397516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575381.845466:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306985402, at 4306985408
00000100:00000010:1.0:1788575381.845483:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000040:0.0:1788575381.845500:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306985402, at 4306985408
00000100:00000010:0.0:1788575381.845522:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575382.293475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575382.293480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575382.293484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575382.293485:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575382.293487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575382.293488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575382.293549:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575382.293554:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575382.293558:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575382.421496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575382.421504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575382.421508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575382.421515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575382.421519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575382.421523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575382.421525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575382.421528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575382.421530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575383.317501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575383.317507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575383.317510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575383.317511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575383.317514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575383.317515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575383.317567:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575383.317570:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575383.317573:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575383.445473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575383.445482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575383.445484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575383.445490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575383.445494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575383.445497:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575383.445500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575383.445502:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575383.445504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575384.341495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575384.341500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575384.341504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575384.341507:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575384.341509:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575384.341512:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575384.341514:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575384.341515:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575384.341517:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575384.341518:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575384.341521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575384.341522:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575384.341523:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575384.341524:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575384.341525:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575384.341525:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575384.341526:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575384.341528:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575384.341587:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575384.341591:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575384.341594:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575384.469516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575384.469523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575384.469525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575384.469531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575384.469535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575384.469538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575384.469540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575384.469543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575384.469545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575385.365506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575385.365514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575385.365520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575385.365522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575385.365526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575385.365528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575385.365568:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575385.365571:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575385.365574:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575385.493484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575385.493491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575385.493492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575385.493496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575385.493498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575385.493501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575385.493503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575385.493505:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575385.493507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575386.005454:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575386.005473:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575386.005476:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575386.005480:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575386.005485:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575386.005493:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12317
00000400:00000200:1.0:1788575386.005517:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575386.005523:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575386.005526:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575386.005529:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005531:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575386.005533:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005535:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575386.005538:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575386.005539:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575386.005542:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575386.005545:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575386.005546:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575386.005548:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575386.005550:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005552:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575386.005553:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005555:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575386.005557:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575386.005560:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005562:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575386.005564:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575386.005566:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575386.005568:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575386.005570:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575386.005573:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575386.005581:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788575386.005584:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575386.005587:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575386.005588:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575386.005591:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575386.005593:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575386.005597:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575386.005619:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788575386.005626:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575386.005636:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575386.005646:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575386.005650:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575386.005655:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575386.005679:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575386.005683:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575386.005687:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575386.005688:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575386.005690:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575386.005693:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575386.005695:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12317
00000400:00000200:1.0:1788575386.005699:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575386.005702:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575386.005720:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575386.005723:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005725:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575386.005727:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005729:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575386.005731:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575386.005733:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575386.005734:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575386.005736:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575386.005738:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575386.005740:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575386.005743:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575386.005743:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788575386.005744:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575386.005748:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575386.005749:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575386.005750:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575386.005754:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1788575386.005755:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575386.005760:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005763:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575386.005764:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1788575386.005765:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788575386.005769:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575386.005774:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.005778:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575386.005780:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575386.005785:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575386.005785:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575386.005789:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:1.0:1788575386.005790:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575386.005794:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575386.005794:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575386.005800:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575386.005802:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393205504:192.168.202.110@tcp:250:kworker.0
00000100:00000010:1.0:1788575386.005804:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788575386.005811:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:3.0:1788575386.005811:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788575386.005815:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575386.005817:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575386.005817:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575386.005820:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575386.005821:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575386.005824:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575386.005825:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1788575386.005827:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575386.005831:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000010:1.0:1788575386.005833:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
02000000:00000001:3.0:1788575386.005834:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575386.005840:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575386.005842:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:1.0:1788575386.005851:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575386.005853:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575386.005862:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000001:1.0:1788575386.005863:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575386.005868:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575386.005870:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393205504, portal 25
00000100:00000001:3.0:1788575386.005875:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000040:1.0:1788575386.005876:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.005878:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00000040:3.0:1788575386.005887:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575411 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575386.005900:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.005901:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575386.005906:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788575386.005907:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1788575386.005908:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575386.005914:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393205504, offset 0
00000100:00080000:1.0:1788575386.005915:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575386.005919:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575386.005923:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575386.005923:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005927:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:3.0:1788575386.005941:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788575386.005942:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12317
00000400:00000200:1.0:1788575386.005952:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575386.005952:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1788575386.005961:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1788575386.005964:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1788575386.005965:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575386.005972:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:1.0:1788575386.005973:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575386.005979:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575386.005984:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:1.0:1788575386.005986:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575386.005990:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330861:330861:256:4294967295] 192.168.202.110@tcp LPNI seq info [330818:330818:8:4294967295]
00000100:00000040:1.0:1788575386.005994:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575386.006000:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575386.006001:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575386.006006:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575386.006009:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575386.006013:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1788575386.006014:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:1.0:1788575386.006018:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:3.0:1788575386.006024:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000001:1.0:1788575386.006025:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000010:3.0:1788575386.006030:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000020:00000001:1.0:1788575386.006031:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575386.006040:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575386.006041:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788575386.006047:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575386.006047:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:1.0:1788575386.006053:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575386.006055:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575386.006059:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575386.006062:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575386.006065:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000010:3.0:1788575386.006066:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000020:00000040:1.0:1788575386.006071:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575386.006076:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:1.0:1788575386.006077:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575386.006083:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575386.006087:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:1.0:1788575386.006088:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575386.006093:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575386.006096:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575386.006097:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000100:00000001:1.0:1788575386.006098:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575386.006102:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000010:3.0:1788575386.006104:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000008:00000001:1.0:1788575386.006105:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575386.006110:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575386.006113:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000100:00000001:3.0:1788575386.006114:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000020:00000040:1.0:1788575386.006116:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575386.006121:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:3.0:1788575386.006121:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575411 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788575386.006123:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575386.006127:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575386.006130:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575386.006133:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006138:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000010:1.0:1788575386.006139:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000040:3.0:1788575386.006144:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575411 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575386.006145:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575386.006155:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575386.006159:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006164:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788575386.006168:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575386.006170:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006173:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575386.006174:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575386.006181:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575386.006186:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306990521
00000100:00000001:1.0:1788575386.006188:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575386.006192:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575386.006195:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:3.0:1788575386.006195:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000040:1.0:1788575386.006202:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12327)
00000100:00000001:3.0:1788575386.006204:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575386.006210:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306990521
00000100:00000010:3.0:1788575386.006213:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575386.006216:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575386.006219:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575386.006221:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006224:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575386.006228:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575386/real 1788575386]  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575386.006236:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575386.006237:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575386.006240:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575386.006241:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575386.006243:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575386.006245:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575386.006247:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575386.006255:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006263:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575386.006265:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006268:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575386.006270:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575386.006275:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006284:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575386.006289:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006298:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575386.006301:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006303:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575386.006305:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006307:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575386.006309:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575386.006312:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575386.006315:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575386.006330:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575386.006333:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575386.006340:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393205504:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575386.006344:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006346:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575386.006348:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393205504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006353:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575386.006354:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575386.006373:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575386.006376:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575386.006377:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575386.006378:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575386.006380:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575386.006382:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:3.0:1788575386.006385:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575386.006386:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575386.006387:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:3.0:1788575386.006390:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575386.006391:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006392:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006398:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575386.006399:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006401:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575386.006402:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575386.006403:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575386.006425:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006432:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575386.006433:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575386.006435:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575386.006436:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575386.006438:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575386.006440:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393205632:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575386.006442:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575386.006444:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575386.006445:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575386.006446:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575386.006449:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575386.006450:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575386.006453:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575386.006455:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575386.006459:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575386.006461:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393205632, portal 10
00000100:00000001:3.0:1788575386.006463:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575386.006464:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:3.0:1788575386.006469:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575411 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006473:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575386.006475:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575386.006476:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575386.006478:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393205632, offset 0
00000400:00000200:3.0:1788575386.006481:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575386.006484:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575386.006486:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575386.006488:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575386.006490:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575386.006492:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575386.006494:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575386.006496:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330862:330862:256:4294967295] 192.168.202.110@tcp LPNI seq info [330819:330819:8:4294967295]
00000400:00000200:3.0:1788575386.006499:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575386.006504:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575386.006508:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575386.006510:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575386.006513:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575386.006515:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575386.006518:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575386.006520:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575386.006523:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575386.006524:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575386.006527:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575386.006529:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575386.006531:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575386.006533:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575386.006534:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575386.006537:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575411 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006542:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575386.006544:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575411 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006548:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006549:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575386.006551:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575386.006554:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575386.006556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575386.006557:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306989607
00000100:00000001:3.0:1788575386.006559:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306990521
00000100:00000001:3.0:1788575386.006560:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306989607
00000100:00000001:3.0:1788575386.006562:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306990521
00000100:00000001:3.0:1788575386.006563:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575386.006565:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006566:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575386.006568:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575386/real 1788575386]  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575386.006575:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575386.006575:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575386.006577:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575386.006578:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575386.006579:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575386.006580:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575386.006582:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575386.006587:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006593:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575386.006595:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006597:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575386.006598:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575386.006602:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006609:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575386.006614:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006621:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575386.006624:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006626:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575386.006627:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006629:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575386.006631:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575386.006634:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575386.006638:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575386.006641:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575386.006645:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575386.006651:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393205632:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575386.006654:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575386.006656:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006660:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575386.006661:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575386.006663:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575386.006664:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575386.006666:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575386.006667:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393205760:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575386.006669:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575386.006671:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575386.006672:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575386.006674:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575386.006678:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575386.006680:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575386.006684:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575386.006687:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575386.006690:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575386.006693:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393205760, portal 4
00000100:00000001:3.0:1788575386.006696:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575386.006697:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000040:3.0:1788575386.006704:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575411 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006712:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575386.006714:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575386.006716:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575386.006719:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393205760, offset 0
00000400:00000200:3.0:1788575386.006723:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575386.006726:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575386.006729:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575386.006731:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575386.006733:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575386.006735:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575386.006737:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575386.006739:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330863:330863:256:4294967295] 192.168.202.110@tcp LPNI seq info [330820:330820:8:4294967295]
00000400:00000200:3.0:1788575386.006741:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575386.006746:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575386.006749:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575386.006752:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575386.006754:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575386.006756:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575386.006759:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575386.006761:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575386.006764:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575386.006765:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575386.006767:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575386.006770:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575386.006772:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575386.006773:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575386.006775:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575386.006777:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575411 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006782:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575386.006784:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575411 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006789:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006790:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575386.006792:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575386.006794:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575386.006796:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575386.006798:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306989753
00000100:00000001:3.0:1788575386.006800:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306990521
00000100:00000001:3.0:1788575386.006801:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306989753
00000100:00000001:3.0:1788575386.006803:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306990521
00000100:00000001:3.0:1788575386.006804:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575386.006806:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006807:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575386.006809:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575386/real 1788575386]  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575386.006814:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575386.006815:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575386.006816:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575386.006818:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575386.006819:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575386.006820:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575386.006822:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575386.006828:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006831:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575386.006832:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006833:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575386.006834:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575386.006836:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006840:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575386.006843:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006861:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575386.006864:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006865:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575386.006866:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006867:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575386.006868:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575386.006870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575386.006872:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575386.006874:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575386.006876:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575386.006881:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393205760:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575386.006883:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006885:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575386.006887:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393205632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006892:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575386.006893:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575386.006894:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575386.006896:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575386.006897:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575386.006898:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575386.006899:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575386.006901:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:3.0:1788575386.006902:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575386.006903:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575386.006904:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:3.0:1788575386.006906:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575386.006907:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006908:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575386.006910:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393205760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575411 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575386.006919:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575386.006920:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575386.006922:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575386.006924:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575386.006926:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575386.006927:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575386.006929:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575386.006930:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:3.0:1788575386.006932:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575386.006934:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575386.006935:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:3.0:1788575386.006937:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575386.006938:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006941:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575386.006944:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575386.006946:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575386.006948:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575386.006950:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575386.389511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575386.389516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575386.389520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575386.389521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575386.389523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575386.389524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575386.389577:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575386.389581:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575386.389585:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575386.517499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575386.517509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575386.517512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575386.517519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575386.517523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575386.517529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575386.517533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575386.517536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575386.517540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575386.965480:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306990521, at 4306990528
00000100:00000040:1.0:1788575386.965485:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306990521, at 4306990528
00000100:00000010:0.0:1788575386.965499:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:1.0:1788575386.965501:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575387.413507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575387.413514:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575387.413516:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575387.413519:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575387.413525:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575387.413527:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575387.413529:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575387.413532:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575387.413536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575387.413538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575387.413542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575387.413543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575387.413621:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575387.413626:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575387.413629:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575387.541536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575387.541543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575387.541545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575387.541549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575387.541552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575387.541555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575387.541557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575387.541559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575387.541561:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575388.437517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575388.437523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575388.437528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575388.437529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575388.437533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575388.437534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575388.437600:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575388.437604:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575388.437607:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575388.565508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575388.565515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575388.565517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575388.565522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575388.565525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575388.565528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575388.565530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575388.565532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575388.565535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575389.461461:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575389.461467:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575389.461474:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575389.461475:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575389.461478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575389.461480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575389.461544:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575389.461548:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575389.461552:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575389.589519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575389.589527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575389.589530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575389.589535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575389.589539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575389.589543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575389.589546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575389.589549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575389.589552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575390.485454:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575390.485460:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575390.485464:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575390.485465:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575390.485468:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575390.485469:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575390.485527:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575390.485531:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575390.485535:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575390.613478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575390.613484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575390.613486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575390.613490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575390.613493:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575390.613496:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575390.613498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575390.613500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575390.613502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575391.125514:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575391.125525:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575391.125528:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575391.125531:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575391.125537:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575391.125544:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12322
00000400:00000200:2.0:1788575391.125562:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575391.125568:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575391.125571:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575391.125574:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575391.125576:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575391.125578:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575391.125580:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575391.125583:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575391.125585:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575391.125588:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575391.125590:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575391.125592:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575391.125593:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575391.125595:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575391.125597:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575391.125598:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575391.125600:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575391.125603:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575391.125605:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575391.125607:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575391.125610:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575391.125612:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575391.125614:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575391.125616:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575391.125618:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575391.125625:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03800.
00000020:00000040:2.0:1788575391.125628:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575391.125631:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575391.125633:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575391.125636:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575391.125639:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575391.125643:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575391.125651:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000001:2.0:1788575391.125658:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575391.125667:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575391.125678:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575391.125682:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575391.125687:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014d98894] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575391.125711:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575391.125715:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575391.125719:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575391.125720:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575391.125722:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575391.125726:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575391.125729:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12322
00000400:00000200:2.0:1788575391.125733:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575391.125737:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575391.125739:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575391.125743:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575391.125745:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575391.125748:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575391.125751:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575391.125754:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575391.125756:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575391.125758:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575391.125761:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575391.125761:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575391.125766:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575391.125767:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575391.125770:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788575391.125772:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575391.125778:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575391.125781:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575391.125788:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575391.125793:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1788575391.125794:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575391.125796:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575391.125799:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575391.125800:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575391.125803:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1788575391.125805:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575391.125806:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575391.125813:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393205888:192.168.202.110@tcp:250:kworker.0
00000100:00080000:2.0:1788575391.125815:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575391.125819:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1788575391.125822:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575391.125823:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575391.125826:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575391.125828:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575391.125829:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575391.125835:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575391.125837:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575391.125837:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575391.125844:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575391.125847:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575391.125849:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1788575391.125852:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000008:00100000:2.0:1788575391.125856:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1788575391.125859:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000008:00000001:2.0:1788575391.125861:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575391.125866:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575391.125866:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393205888, portal 25
00000100:00000010:2.0:1788575391.125871:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01f80.
00000020:00000040:2.0:1788575391.125875:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575391.125878:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1788575391.125878:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575391.125880:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
02000000:00000001:2.0:1788575391.125883:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575391.125887:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575391.125891:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1788575391.125892:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575391.125896:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.125903:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000010:2.0:1788575391.125905:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000040:3.0:1788575391.125907:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575391.125912:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000001:2.0:1788575391.125914:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575391.125916:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393205888, offset 0
00000400:00000200:3.0:1788575391.125925:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00100000:2.0:1788575391.125932:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575391.125932:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575391.125942:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575391.125943:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1788575391.125946:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575391.125952:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1788575391.125953:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8c8ec7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575391.125961:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1788575391.125964:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575391.125968:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1788575391.125970:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575391.125974:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:2.0:1788575391.125977:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575391.125982:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1788575391.125983:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330864:330864:256:4294967295] 192.168.202.110@tcp LPNI seq info [330821:330821:8:4294967295]
00000100:00080000:2.0:1788575391.125985:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575391.125993:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575391.126000:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575391.126007:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12322
00000400:00000200:3.0:1788575391.126012:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1788575391.126017:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575391.126026:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:3.0:1788575391.126026:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575391.126032:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000010:3.0:1788575391.126036:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00000040:2.0:1788575391.126037:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575391.126041:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575391.126043:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575391.126045:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:3.0:1788575391.126048:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1788575391.126049:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575391.126051:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575391.126054:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.0:1788575391.126056:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000040:2.0:1788575391.126057:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575391.126060:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575391.126064:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575391.126069:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575391.126069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575391.126073:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575391.126077:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575391.126077:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1788575391.126079:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575391.126085:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575391.126089:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000100:3.0:1788575391.126091:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575391.126092:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575391.126096:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:3.0:1788575391.126098:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
02000000:00000001:2.0:1788575391.126099:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575391.126103:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575391.126104:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575391.126106:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:3.0:1788575391.126107:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00000001:2.0:1788575391.126108:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575391.126113:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575391.126117:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00a80.
00000400:00000200:3.0:1788575391.126120:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:2.0:1788575391.126121:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575391.126126:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575391.126128:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575391.126130:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575391.126131:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575391.126131:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
02000000:00000001:2.0:1788575391.126135:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575391.126136:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
02000000:00000010:2.0:1788575391.126139:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000100:00000001:2.0:1788575391.126143:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.126145:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00100000:2.0:1788575391.126149:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575391.126155:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575391.126161:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575391.126165:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.126172:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575391.126173:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000389a8694] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575391.126178:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575416 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575391.126193:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.126195:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575391.126198:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575391.126199:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:2.0:1788575391.126203:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575391.126206:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575391.126208:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12332)
00000400:00000200:3.0:1788575391.126215:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575391.126219:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4306995641
00000100:00000010:3.0:1788575391.126226:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575391.126233:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575391.126276:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4306995641
00000100:00000010:3.0:1788575391.126282:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575391.126286:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575391.126290:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575391.126293:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.126296:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575391.126303:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575391/real 1788575391]  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575391.126314:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575391.126316:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575391.126319:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575391.126323:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575391.126324:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575391.126327:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575391.126331:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575391.126337:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126345:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575391.126347:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126349:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575391.126351:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575391.126353:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126358:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575391.126361:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126366:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575391.126368:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575391.126370:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126371:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575391.126373:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575391.126375:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575391.126377:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575391.126379:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575391.126381:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575391.126386:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393205888:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575391.126412:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126414:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575391.126417:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393205888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126421:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575391.126423:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575391.126426:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575391.126428:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575391.126429:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575391.126430:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575391.126432:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575391.126434:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575391.126437:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575391.126438:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575391.126439:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:3.0:1788575391.126441:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575391.126442:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126444:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126450:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575391.126451:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.126453:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575391.126454:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575391.126455:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575391.126457:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126462:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575391.126463:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575391.126465:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575391.126466:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575391.126468:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575391.126470:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206016:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575391.126472:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575391.126474:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575391.126475:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575391.126476:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575391.126478:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575391.126480:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575391.126483:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575391.126485:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575391.126487:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575391.126489:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393206016, portal 10
00000100:00000001:3.0:1788575391.126492:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575391.126493:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:3.0:1788575391.126497:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126502:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575391.126503:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575391.126505:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575391.126506:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393206016, offset 0
00000400:00000200:3.0:1788575391.126509:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575391.126512:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575391.126514:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575391.126517:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575391.126519:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575391.126521:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575391.126522:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575391.126525:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330865:330865:256:4294967295] 192.168.202.110@tcp LPNI seq info [330822:330822:8:4294967295]
00000400:00000200:3.0:1788575391.126527:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575391.126533:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575391.126536:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575391.126538:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575391.126541:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575391.126543:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788575391.126546:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575391.126548:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1788575391.126551:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575391.126553:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575391.126554:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575391.126557:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575391.126559:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575391.126560:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575391.126562:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575391.126564:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126574:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575391.126576:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575416 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126580:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.126581:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575391.126583:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575391.126585:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575391.126587:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575391.126589:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306994727
00000100:00000001:3.0:1788575391.126591:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306995641
00000100:00000001:3.0:1788575391.126592:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306994727
00000100:00000001:3.0:1788575391.126594:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306995641
00000100:00000001:3.0:1788575391.126595:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575391.126596:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.126598:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575391.126600:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575391/real 1788575391]  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575391.126605:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575391.126606:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575391.126607:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575391.126608:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575391.126609:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575391.126610:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575391.126612:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575391.126617:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126621:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575391.126623:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126624:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575391.126625:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575391.126628:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126632:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575391.126634:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126638:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575391.126640:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126641:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575391.126641:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126643:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575391.126644:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575391.126646:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575391.126647:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575391.126649:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575391.126651:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575391.126657:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206016:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575391.126659:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575391.126661:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126665:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575391.126666:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575391.126667:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575391.126668:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575391.126669:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575391.126672:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575391.126674:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575391.126675:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575391.126676:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575391.126677:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575391.126680:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575391.126681:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575391.126683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575391.126684:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575391.126686:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575391.126688:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393206144, portal 4
00000100:00000001:3.0:1788575391.126689:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575391.126690:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:3.0:1788575391.126695:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126702:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575391.126705:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575391.126707:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575391.126710:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393206144, offset 0
00000400:00000200:3.0:1788575391.126714:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575391.126718:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575391.126722:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575391.126725:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575391.126727:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575391.126729:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575391.126732:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575391.126735:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330866:330866:256:4294967295] 192.168.202.110@tcp LPNI seq info [330823:330823:8:4294967295]
00000400:00000200:3.0:1788575391.126739:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575391.126746:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575391.126751:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575391.126755:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575391.126760:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575391.126763:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788575391.126767:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575391.126770:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1788575391.126774:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575391.126776:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575391.126780:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575391.126783:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575391.126787:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575391.126789:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575391.126791:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575391.126795:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126804:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575391.126808:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575416 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126815:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.126817:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575391.126821:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575391.126824:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575391.126829:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575391.126831:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306994873
00000100:00000001:3.0:1788575391.126834:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4306995641
00000100:00000001:3.0:1788575391.126836:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306994873
00000100:00000001:3.0:1788575391.126840:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4306995641
00000100:00000001:3.0:1788575391.126842:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575391.126844:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.126846:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575391.126851:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575391/real 1788575391]  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575391.126859:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575391.126861:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575391.126863:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575391.126866:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575391.126868:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575391.126870:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575391.126874:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575391.126883:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126891:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575391.126893:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126895:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575391.126896:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575391.126900:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126907:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575391.126912:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126920:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575391.126922:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126924:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575391.126926:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126928:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575391.126930:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575391.126933:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575391.126937:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575391.126939:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575391.126945:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575391.126961:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575391.126965:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.126967:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575391.126971:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393206016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.126978:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575391.126980:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575391.126982:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575391.126985:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575391.126986:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575391.126988:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575391.126990:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575391.126992:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575391.126994:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575391.126995:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575391.126997:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575391.127000:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575391.127002:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.127003:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575391.127007:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393206144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575416 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575391.127013:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575391.127014:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575391.127015:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575391.127017:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575391.127017:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575391.127018:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575391.127020:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575391.127020:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575391.127022:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575391.127023:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575391.127023:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575391.127025:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575391.127026:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.127028:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575391.127030:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575391.127031:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575391.127033:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575391.127034:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575391.509462:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575391.509468:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575391.509472:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575391.509473:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575391.509475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575391.509476:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575391.509531:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575391.509534:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575391.509537:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575391.637511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575391.637518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575391.637520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575391.637525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575391.637527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575391.637531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575391.637533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575391.637535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575391.637538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575392.085516:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4306995641, at 4306995648
00000100:00000040:1.0:1788575392.085529:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4306995641, at 4306995648
00000100:00000010:0.0:1788575392.085534:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:1.0:1788575392.085538:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575392.533548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575392.533556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575392.533563:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575392.533565:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575392.533569:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575392.533571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575392.533633:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575392.533637:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575392.533639:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575392.661506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575392.661512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575392.661514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575392.661524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575392.661527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575392.661534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575392.661537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575392.661539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575392.661542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575393.557502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575393.557509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575393.557515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575393.557517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575393.557521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575393.557522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575393.557589:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575393.557593:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575393.557596:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575393.685538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575393.685545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575393.685548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575393.685559:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575393.685563:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575393.685565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575393.685568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575393.685570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575393.685572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575394.581466:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575394.581472:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575394.581476:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575394.581478:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575394.581480:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575394.581482:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575394.581484:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575394.581486:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575394.581487:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575394.581489:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575394.581491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575394.581492:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575394.581492:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575394.581494:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575394.581495:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575394.581496:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575394.581498:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575394.581499:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575394.581571:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575394.581574:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575394.581577:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575394.709493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575394.709498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575394.709500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575394.709504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575394.709506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575394.709510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575394.709512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575394.709514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575394.709516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575395.605498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575395.605503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575395.605506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575395.605507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575395.605510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575395.605511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575395.605565:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575395.605569:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575395.605571:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575395.733515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575395.733523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575395.733525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575395.733531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575395.733536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575395.733540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575395.733543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575395.733545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575395.733548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575396.245515:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575396.245524:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575396.245526:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575396.245529:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575396.245534:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575396.245540:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12327
00000400:00000200:2.0:1788575396.245548:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575396.245553:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575396.245556:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575396.245559:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575396.245562:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575396.245565:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575396.245566:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575396.245569:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575396.245571:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575396.245573:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575396.245575:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575396.245577:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575396.245579:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575396.245581:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575396.245583:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575396.245586:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575396.245587:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575396.245591:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575396.245594:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575396.245597:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575396.245601:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575396.245604:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575396.245607:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575396.245610:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575396.245612:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575396.245622:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01c00.
00000020:00000040:2.0:1788575396.245626:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575396.245631:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575396.245633:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575396.245635:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575396.245638:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575396.245643:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575396.245650:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000100:00000001:2.0:1788575396.245657:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575396.245666:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575396.245676:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575396.245681:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575396.245686:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000de3acaa1] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575396.245706:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575396.245709:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575396.245713:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575396.245714:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575396.245716:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575396.245719:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575396.245723:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12327
00000400:00000200:2.0:1788575396.245728:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575396.245732:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575396.245735:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575396.245737:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1788575396.245739:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575396.245742:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575396.245743:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575396.245745:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575396.245750:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575396.245754:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575396.245755:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575396.245761:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575396.245764:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575396.245767:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1788575396.245768:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575396.245770:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575396.245773:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575396.245777:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575396.245779:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:3.0:1788575396.245780:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575396.245784:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575396.245785:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575396.245786:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575396.245792:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1788575396.245793:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393206272:192.168.202.110@tcp:250:kworker.0
00000100:00000040:2.0:1788575396.245799:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575396.245800:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575396.245804:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1788575396.245805:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575396.245807:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575396.245810:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575396.245810:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:2.0:1788575396.245816:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000010:3.0:1788575396.245817:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
00000100:00000001:2.0:1788575396.245820:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575396.245822:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575396.245824:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575396.245832:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575396.245833:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575396.245838:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575396.245840:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575396.245844:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1788575396.245846:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000008:00000001:2.0:1788575396.245848:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575396.245853:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575396.245856:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000010:2.0:1788575396.245858:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00e00.
00000020:00000040:2.0:1788575396.245863:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000200:3.0:1788575396.245866:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393206272, portal 25
02000000:00000001:2.0:1788575396.245867:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575396.245870:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575396.245873:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1788575396.245873:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575396.245876:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
02000000:00000001:2.0:1788575396.245877:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575396.245882:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575396.245886:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788575396.245889:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788575396.245896:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.245897:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575396.245902:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575396.245905:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00100000:2.0:1788575396.245907:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575396.245909:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393206272, offset 0
00000400:00000200:3.0:1788575396.245920:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575396.245924:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575396.245928:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575396.245933:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575396.245934:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5f665f0] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575396.245945:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575396.245952:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788575396.245957:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575396.245958:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1788575396.245964:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575396.245965:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1788575396.245972:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:3.0:1788575396.245973:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575396.245977:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1788575396.245980:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330867:330867:256:4294967295] 192.168.202.110@tcp LPNI seq info [330824:330824:8:4294967295]
00000100:00080000:2.0:1788575396.245982:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575396.245987:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575396.245988:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575396.245993:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12327
00000400:00000200:3.0:1788575396.246003:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1788575396.246004:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:3.0:1788575396.246012:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575396.246014:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:3.0:1788575396.246018:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00080000:2.0:1788575396.246019:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575396.246026:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575396.246027:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575396.246032:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575396.246033:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:2.0:1788575396.246039:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575396.246043:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575396.246044:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:3.1:1788575396.246046:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:2.0:1788575396.246051:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000100:3.0:1788575396.246052:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:2.0:1788575396.246056:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000010:3.0:1788575396.246056:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000020:00000001:2.0:1788575396.246061:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1788575396.246063:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1788575396.246065:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575396.246069:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1788575396.246071:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575396.246075:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000020:00000001:2.0:1788575396.246076:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:3.0:1788575396.246077:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575396.246085:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:2.0:1788575396.246086:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:3.0:1788575396.246090:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575396.246099:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575396.246100:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575396.246107:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575396.246108:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575396.246114:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575396.246121:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:3.0:1788575396.246128:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246131:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00080000:2.0:1788575396.246132:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575396.246136:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575396.246138:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575396.246142:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575396.246144:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575396.246146:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307000761
02000000:00000001:2.0:1788575396.246150:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:3.0:1788575396.246152:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
02000000:00000001:2.0:1788575396.246156:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246160:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575396.246161:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575396.246166:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:3.0:1788575396.246167:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307000761
00000008:00000001:2.0:1788575396.246188:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575396.246191:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575396.246195:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03100.
00000100:00000010:3.0:1788575396.246197:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000020:00000040:2.0:1788575396.246199:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:3.0:1788575396.246201:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:2.0:1788575396.246205:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575396.246208:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246208:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575396.246221:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575396.246224:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575396.246227:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575396.246230:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246230:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575396.246239:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575396/real 1788575396]  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788575396.246242:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000001:2.0:1788575396.246248:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575396.246259:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:3.0:1788575396.246260:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575396.246264:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575396.246268:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575396.246274:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000100:00000001:2.0:1788575396.246275:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1788575396.246279:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:2.0:1788575396.246280:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246285:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000040:2.0:1788575396.246289:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008692924c] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575396.246294:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575396.246318:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575396.246320:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575396.246324:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575396.246328:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:2.0:1788575396.246332:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575396.246332:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246334:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575396.246337:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575396.246342:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12337)
00000100:00100000:3.0:1788575396.246342:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246352:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575396.246358:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246366:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575396.246369:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246371:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575396.246372:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246375:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575396.246377:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575396.246380:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575396.246385:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575396.246388:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575396.246415:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575396.246422:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393206272:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575396.246426:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246427:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575396.246430:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393206272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246434:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575396.246436:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575396.246439:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575396.246441:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575396.246442:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575396.246444:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575396.246445:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575396.246447:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575396.246450:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575396.246452:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575396.246453:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:3.0:1788575396.246455:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575396.246456:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246458:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246464:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575396.246465:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246466:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575396.246467:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575396.246468:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575396.246471:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246476:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575396.246477:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575396.246479:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575396.246480:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575396.246482:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575396.246483:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575396.246485:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575396.246487:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575396.246488:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575396.246489:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575396.246492:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575396.246493:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575396.246496:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575396.246498:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575396.246501:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575396.246503:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393206400, portal 10
00000100:00000001:3.0:1788575396.246505:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575396.246506:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:3.0:1788575396.246511:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246515:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575396.246517:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575396.246519:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575396.246520:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393206400, offset 0
00000400:00000200:3.0:1788575396.246524:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575396.246527:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575396.246529:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575396.246532:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575396.246534:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575396.246535:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575396.246537:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575396.246539:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330868:330868:256:4294967295] 192.168.202.110@tcp LPNI seq info [330825:330825:8:4294967295]
00000400:00000200:3.0:1788575396.246542:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575396.246547:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575396.246550:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575396.246552:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575396.246555:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575396.246557:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575396.246560:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575396.246562:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575396.246564:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575396.246566:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575396.246568:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575396.246570:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575396.246581:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575396.246583:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575396.246586:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575396.246590:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246598:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575396.246601:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246609:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246610:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575396.246615:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575396.246618:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575396.246622:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575396.246624:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306999847
00000100:00000001:3.0:1788575396.246628:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307000761
00000100:00000001:3.0:1788575396.246629:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306999847
00000100:00000001:3.0:1788575396.246631:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307000761
00000100:00000001:3.0:1788575396.246632:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575396.246634:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246635:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575396.246638:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575396/real 1788575396]  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575396.246644:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575396.246645:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575396.246646:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575396.246648:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575396.246649:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575396.246651:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575396.246653:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575396.246658:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246663:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575396.246664:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246665:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575396.246666:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575396.246668:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246672:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575396.246674:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246679:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575396.246681:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246682:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575396.246683:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246684:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575396.246686:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575396.246687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575396.246689:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575396.246691:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575396.246693:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575396.246698:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575396.246700:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575396.246703:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246707:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575396.246708:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575396.246709:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575396.246711:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575396.246712:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575396.246714:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206528:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575396.246716:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575396.246717:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575396.246718:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575396.246719:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575396.246722:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575396.246724:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575396.246726:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575396.246728:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575396.246730:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575396.246732:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393206528, portal 4
00000100:00000001:3.0:1788575396.246734:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575396.246735:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:3.0:1788575396.246739:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246743:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575396.246745:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575396.246746:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575396.246748:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393206528, offset 0
00000400:00000200:3.0:1788575396.246750:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575396.246752:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575396.246755:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575396.246757:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575396.246759:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575396.246760:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575396.246762:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575396.246764:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330869:330869:256:4294967295] 192.168.202.110@tcp LPNI seq info [330826:330826:8:4294967295]
00000400:00000200:3.0:1788575396.246767:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575396.246772:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575396.246775:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575396.246777:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575396.246779:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575396.246781:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575396.246784:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575396.246786:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575396.246789:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575396.246790:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575396.246792:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575396.246795:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575396.246797:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575396.246798:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575396.246800:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575396.246802:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246807:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575396.246809:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575421 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246814:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246815:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575396.246817:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575396.246819:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575396.246822:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575396.246824:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4306999993
00000100:00000001:3.0:1788575396.246827:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307000761
00000100:00000001:3.0:1788575396.246829:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4306999993
00000100:00000001:3.0:1788575396.246831:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307000761
00000100:00000001:3.0:1788575396.246834:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575396.246836:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246838:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575396.246842:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575396/real 1788575396]  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575396.246851:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575396.246852:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575396.246854:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575396.246857:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575396.246859:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575396.246861:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575396.246865:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575396.246874:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246884:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575396.246885:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246886:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575396.246887:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575396.246890:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246894:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575396.246897:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246901:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575396.246903:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246904:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575396.246905:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246906:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575396.246907:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575396.246909:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575396.246911:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575396.246912:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575396.246915:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575396.246920:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206528:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575396.246922:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246924:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575396.246927:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393206400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246934:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575396.246935:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575396.246937:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575396.246940:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575396.246942:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575396.246943:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575396.246945:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575396.246947:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575396.246950:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575396.246951:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575396.246953:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:3.0:1788575396.246956:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575396.246958:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246960:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575396.246965:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393206528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575421 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575396.246973:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575396.246975:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575396.246977:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575396.246979:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575396.246980:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575396.246981:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575396.246982:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575396.246983:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575396.246985:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575396.246986:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575396.246987:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:3.0:1788575396.246989:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575396.246990:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246991:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575396.246994:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575396.246995:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575396.246996:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575396.246998:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575396.629495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575396.629501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575396.629504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575396.629505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575396.629508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575396.629509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575396.629568:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575396.629571:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575396.629575:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575396.757478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575396.757484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575396.757486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575396.757491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575396.757495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575396.757499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575396.757501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575396.757504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575396.757506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575397.205495:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307000761, at 4307000768
00000100:00000040:1.0:1788575397.205495:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307000761, at 4307000768
00000100:00000010:1.0:1788575397.205512:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:0.0:1788575397.205518:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00010000:00000001:3.0:1788575397.653512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575397.653520:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575397.653523:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575397.653526:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575397.653532:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575397.653534:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575397.653537:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575397.653539:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575397.653543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575397.653545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575397.653549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575397.653550:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575397.653615:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575397.653620:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575397.653623:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575397.781556:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575397.781564:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575397.781567:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575397.781573:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575397.781576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575397.781580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575397.781583:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575397.781586:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575397.781589:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575398.677518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575398.677525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575398.677534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575398.677535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575398.677539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575398.677540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575398.677597:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575398.677602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575398.677605:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575398.805525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575398.805532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575398.805534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575398.805540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575398.805544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575398.805548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575398.805551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575398.805553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575398.805557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575399.701488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575399.701493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575399.701497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575399.701498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575399.701500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575399.701501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575399.701524:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575399.701528:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575399.701532:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575399.829559:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575399.829566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575399.829568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575399.829573:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575399.829576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575399.829580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575399.829582:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575399.829584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575399.829587:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575400.725491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575400.725496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575400.725499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575400.725500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575400.725502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575400.725503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575400.725552:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575400.725555:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575400.725558:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575400.853477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575400.853485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575400.853487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575400.853493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575400.853497:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575400.853500:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575400.853503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575400.853507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575400.853510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575401.365476:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575401.365486:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575401.365489:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575401.365493:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575401.365497:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575401.365504:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12332
00000400:00000200:3.0:1788575401.365513:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575401.365519:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575401.365522:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575401.365525:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365528:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575401.365530:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365533:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575401.365536:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575401.365538:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575401.365541:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575401.365544:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575401.365546:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575401.365549:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575401.365551:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365553:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575401.365555:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365561:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575401.365565:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575401.365568:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365570:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575401.365573:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575401.365577:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575401.365579:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575401.365582:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575401.365584:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575401.365593:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000020:00000040:3.0:1788575401.365597:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575401.365600:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575401.365602:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575401.365605:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575401.365607:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575401.365611:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575401.365618:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575401.365625:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575401.365633:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575401.365644:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575401.365647:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575401.365652:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575401.365674:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575401.365677:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575401.365680:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575401.365682:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575401.365683:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575401.365686:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575401.365689:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12332
00000400:00000200:3.0:1788575401.365693:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575401.365697:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575401.365699:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575401.365702:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575401.365703:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365707:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575401.365708:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575401.365710:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575401.365713:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575401.365714:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575401.365717:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575401.365719:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575401.365721:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575401.365724:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575401.365724:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575401.365727:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575401.365729:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575401.365732:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365736:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575401.365740:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365744:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575401.365749:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575401.365750:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575401.365754:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575401.365754:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575401.365758:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575401.365759:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575401.365762:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575401.365763:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575401.365766:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575401.365770:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393206656:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575401.365770:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575401.365776:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575401.365776:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575401.365780:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575401.365782:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575401.365785:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575401.365785:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575401.365790:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:3.0:1788575401.365791:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575401.365794:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575401.365795:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575401.365798:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575401.365799:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575401.365802:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:2.0:1788575401.365804:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575401.365808:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575401.365810:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575401.365812:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575401.365813:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575401.365815:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575401.365816:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
02000000:00000010:3.0:1788575401.365820:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000400:00000010:2.0:1788575401.365825:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000001:3.0:1788575401.365825:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575401.365834:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788575401.365835:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393206656, portal 25
00000100:00000001:2.0:1788575401.365841:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575401.365844:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000001:3.0:1788575401.365846:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575401.365850:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575401.365853:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575401.365857:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.365865:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575401.365872:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365874:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575401.365879:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575401.365882:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:2.0:1788575401.365884:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000001:3.0:1788575401.365884:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575401.365888:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:2.0:1788575401.365890:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393206656, offset 0
00000100:00000001:3.0:1788575401.365891:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575401.365896:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12332
00000400:00000200:3.0:1788575401.365901:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575401.365902:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575401.365906:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575401.365909:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575401.365912:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365915:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575401.365921:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365923:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575401.365925:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1788575401.365926:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575401.365928:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575401.365931:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575401.365933:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575401.365936:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575401.365937:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:3.0:1788575401.365939:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575401.365944:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575401.365945:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788575401.365947:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575401.365951:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575401.365954:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575401.365954:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575401.365959:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575401.365960:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1788575401.365964:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575401.365965:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575401.365970:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575401.365971:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330870:330870:256:4294967295] 192.168.202.110@tcp LPNI seq info [330827:330827:8:4294967295]
02000000:00000001:3.0:1788575401.365975:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575401.365978:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575401.365980:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575401.365983:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575401.365988:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788575401.365991:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000008:00100000:3.0:1788575401.365991:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575401.365996:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575401.365998:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575401.366000:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575401.366004:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000100:00000010:3.0:1788575401.366005:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575401.366009:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.0:1788575401.366013:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:3.0:1788575401.366014:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575401.366017:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575401.366020:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575401.366021:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575401.366023:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575401.366024:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
02000000:00000010:3.0:1788575401.366027:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575401.366031:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575401.366031:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575401.366036:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00100000:3.0:1788575401.366041:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:2.0:1788575401.366042:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575401.366046:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000100:00000001:3.0:1788575401.366054:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575401.366055:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1788575401.366057:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575401.366064:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575401.366065:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575401.366075:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000100:00000001:3.0:1788575401.366076:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575401.366079:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000040:3.0:1788575401.366080:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575401.366086:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:3.0:1788575401.366086:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575401.366092:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12342)
00000100:00000200:2.0:1788575401.366093:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366102:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575401.366104:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366109:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575401.366111:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575401.366121:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575401.366149:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575401.366151:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307005881
00000100:00000010:2.0:1788575401.366156:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c900.
00000100:00000001:2.0:1788575401.366172:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575401.366177:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307005881
00000100:00000010:2.0:1788575401.366179:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c540.
00000100:00000001:2.0:1788575401.366182:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575401.366185:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575401.366187:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575401.366188:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575401.366192:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575401/real 1788575401]  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575401.366199:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575401.366201:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575401.366202:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575401.366204:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575401.366205:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575401.366207:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575401.366209:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575401.366220:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366227:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575401.366229:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366231:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575401.366233:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575401.366237:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366245:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575401.366249:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366256:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575401.366259:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366261:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575401.366262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366264:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575401.366266:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575401.366268:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575401.366270:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575401.366272:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575401.366275:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575401.366280:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393206656:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575401.366283:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366284:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575401.366286:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393206656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366289:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575401.366291:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575401.366293:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575401.366295:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575401.366296:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575401.366297:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575401.366298:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575401.366300:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575401.366302:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575401.366303:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575401.366304:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:2.0:1788575401.366306:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575401.366307:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366308:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366314:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575401.366315:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575401.366316:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575401.366317:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575401.366318:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575401.366320:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366328:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575401.366329:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575401.366330:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575401.366332:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575401.366333:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575401.366334:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206784:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575401.366336:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575401.366337:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575401.366338:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575401.366339:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575401.366341:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575401.366342:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575401.366345:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575401.366346:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575401.366349:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575401.366350:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393206784, portal 10
00000100:00000001:2.0:1788575401.366352:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575401.366353:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:2.0:1788575401.366358:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366364:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575401.366367:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575401.366369:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575401.366371:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393206784, offset 0
00000400:00000200:2.0:1788575401.366375:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575401.366379:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575401.366383:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575401.366386:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575401.366388:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575401.366407:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575401.366410:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575401.366412:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330871:330871:256:4294967295] 192.168.202.110@tcp LPNI seq info [330828:330828:8:4294967295]
00000400:00000200:2.0:1788575401.366415:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575401.366419:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575401.366422:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575401.366425:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575401.366428:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575401.366430:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788575401.366432:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575401.366434:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1788575401.366442:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575401.366444:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575401.366446:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575401.366448:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575401.366450:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575401.366451:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575401.366452:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575401.366455:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366459:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575401.366461:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366465:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575401.366466:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575401.366468:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575401.366471:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575401.366473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575401.366474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307004967
00000100:00000001:2.0:1788575401.366476:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307005881
00000100:00000001:2.0:1788575401.366477:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307004967
00000100:00000001:2.0:1788575401.366479:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307005881
00000100:00000001:2.0:1788575401.366480:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575401.366481:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575401.366482:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575401.366484:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575401/real 1788575401]  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575401.366489:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575401.366490:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575401.366491:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575401.366492:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575401.366493:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575401.366494:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575401.366496:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575401.366501:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366504:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575401.366505:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366506:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575401.366507:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575401.366508:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366512:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575401.366514:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366518:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575401.366519:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366520:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575401.366521:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366522:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575401.366532:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575401.366535:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575401.366538:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575401.366540:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575401.366544:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575401.366553:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206784:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575401.366557:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575401.366560:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366566:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575401.366568:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575401.366570:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575401.366571:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575401.366573:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575401.366576:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206912:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575401.366579:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575401.366581:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575401.366582:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575401.366584:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575401.366587:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575401.366589:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575401.366593:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575401.366596:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575401.366599:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575401.366602:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393206912, portal 4
00000100:00000001:2.0:1788575401.366604:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575401.366605:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788575401.366610:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366614:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575401.366616:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575401.366617:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575401.366619:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393206912, offset 0
00000400:00000200:2.0:1788575401.366621:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575401.366623:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575401.366626:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575401.366628:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575401.366629:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575401.366631:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575401.366632:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575401.366634:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330872:330872:256:4294967295] 192.168.202.110@tcp LPNI seq info [330829:330829:8:4294967295]
00000400:00000200:2.0:1788575401.366637:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575401.366641:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575401.366643:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575401.366645:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575401.366648:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575401.366649:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788575401.366652:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575401.366653:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1788575401.366656:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575401.366657:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575401.366659:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575401.366661:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575401.366663:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575401.366664:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575401.366665:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575401.366667:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366672:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575401.366674:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366678:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575401.366679:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575401.366680:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575401.366682:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575401.366684:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575401.366686:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307005113
00000100:00000001:2.0:1788575401.366687:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307005881
00000100:00000001:2.0:1788575401.366688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307005113
00000100:00000001:2.0:1788575401.366690:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307005881
00000100:00000001:2.0:1788575401.366691:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575401.366692:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575401.366693:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575401.366695:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575401/real 1788575401]  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575401.366701:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575401.366702:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575401.366704:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575401.366705:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575401.366707:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575401.366708:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575401.366711:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575401.366719:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366725:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575401.366726:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366728:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575401.366729:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575401.366733:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366739:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575401.366744:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366748:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575401.366750:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575401.366752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366753:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575401.366754:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575401.366755:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575401.366757:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575401.366758:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575401.366761:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575401.366765:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393206912:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575401.366767:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366769:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575401.366770:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393206784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366774:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575401.366775:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575401.366776:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575401.366777:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575401.366778:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575401.366779:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575401.366780:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575401.366781:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575401.366783:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575401.366784:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575401.366785:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575401.366787:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575401.366788:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366789:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575401.366790:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393206912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575401.366794:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575401.366795:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575401.366796:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575401.366797:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575401.366797:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575401.366798:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575401.366799:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575401.366800:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575401.366801:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575401.366801:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575401.366802:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575401.366803:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575401.366804:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366804:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575401.366807:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575401.366807:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575401.366808:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575401.366809:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575401.749467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575401.749483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575401.749488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575401.749490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575401.749493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575401.749495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575401.749534:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575401.749538:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575401.749542:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575401.877443:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575401.877450:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575401.877452:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575401.877457:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575401.877461:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575401.877464:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575401.877473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575401.877476:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575401.877479:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575402.325465:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307005881, at 4307005888
00000100:00000040:0.0:1788575402.325466:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307005881, at 4307005888
00000100:00000010:0.0:1788575402.325479:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000100:00000010:1.0:1788575402.325480:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00010000:00000001:3.0:1788575402.773508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575402.773514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575402.773519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575402.773520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575402.773523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575402.773524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575402.773574:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575402.773578:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575402.773581:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575402.901493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575402.901499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575402.901500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575402.901503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575402.901506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575402.901509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575402.901510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575402.901512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575402.901514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575403.797483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575403.797489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575403.797493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575403.797496:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575403.797497:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575403.797499:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575403.797501:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575403.797502:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575403.797504:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575403.797505:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575403.797507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575403.797508:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575403.797509:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575403.797510:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575403.797510:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575403.797511:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575403.797512:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575403.797514:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575403.797568:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575403.797571:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575403.797573:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575403.925505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575403.925511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575403.925513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575403.925523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575403.925527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575403.925531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575403.925534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575403.925536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575403.925538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788575404.437523:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788575404.437535:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788575404.437539:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788575404.437692:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1788575404.437704:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788575404.437707:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788575404.437713:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575404.437716:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575404.821483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575404.821488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575404.821492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575404.821493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575404.821495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575404.821495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575404.821544:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575404.821547:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575404.821549:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575404.949464:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575404.949470:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575404.949472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575404.949476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575404.949479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575404.949482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575404.949485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575404.949487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575404.949489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575405.845523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575405.845530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575405.845535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575405.845536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575405.845539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575405.845541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575405.845609:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575405.845613:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575405.845616:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575405.973513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575405.973520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575405.973522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575405.973526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575405.973529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575405.973531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575405.973533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575405.973535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575405.973537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575406.485456:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575406.485466:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575406.485469:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575406.485472:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575406.485476:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575406.485482:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12337
00000400:00000200:3.0:1788575406.485490:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575406.485497:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575406.485500:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575406.485504:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485507:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575406.485510:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485512:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575406.485516:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575406.485518:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575406.485521:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575406.485524:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575406.485526:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575406.485528:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575406.485530:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485532:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575406.485533:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485534:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575406.485537:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575406.485539:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485540:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575406.485542:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575406.485545:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575406.485546:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575406.485548:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575406.485550:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575406.485556:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000020:00000040:3.0:1788575406.485558:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575406.485561:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575406.485562:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575406.485564:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575406.485566:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575406.485569:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575406.485574:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575406.485580:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575406.485585:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575406.485593:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575406.485596:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575406.485599:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575406.485616:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575406.485618:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575406.485620:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575406.485621:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575406.485622:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575406.485624:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575406.485626:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12337
00000400:00000200:3.0:1788575406.485628:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575406.485630:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575406.485632:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575406.485634:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485635:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575406.485637:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485639:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575406.485641:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575406.485642:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575406.485644:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575406.485646:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575406.485647:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575406.485648:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575406.485651:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575406.485651:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575406.485654:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575406.485655:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485658:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575406.485662:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485664:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575406.485665:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:2.0:1788575406.485666:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575406.485669:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485673:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575406.485675:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575406.485680:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575406.485683:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1788575406.485683:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575406.485685:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575406.485687:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575406.485689:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575406.485692:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575406.485693:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575406.485693:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575406.485698:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:3.0:1788575406.485698:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788575406.485702:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575406.485703:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575406.485706:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575406.485708:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575406.485709:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393207040:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575406.485710:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575406.485712:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575406.485715:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575406.485716:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:3.0:1788575406.485719:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575406.485720:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575406.485723:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575406.485724:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575406.485726:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:3.0:1788575406.485730:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:2.0:1788575406.485731:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575406.485735:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575406.485741:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575406.485744:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575406.485744:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575406.485750:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000040:3.0:1788575406.485750:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575406.485758:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000001:3.0:1788575406.485759:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575406.485762:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575406.485764:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000200:2.0:1788575406.485765:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393207040, portal 25
00000100:00000001:3.0:1788575406.485765:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1788575406.485769:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1788575406.485769:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575406.485772:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000001:3.0:1788575406.485773:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575406.485778:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12337
00000100:00000040:2.0:1788575406.485780:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575406.485783:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575406.485788:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575406.485792:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575406.485796:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575406.485797:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485801:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575406.485803:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575406.485805:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485806:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575406.485809:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575406.485810:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000010:2.0:1788575406.485812:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000020:00000001:3.0:1788575406.485812:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575406.485814:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575406.485816:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:2.0:1788575406.485818:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393207040, offset 0
00000020:00000001:3.0:1788575406.485818:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575406.485824:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485825:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575406.485827:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485830:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575406.485831:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575406.485832:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575406.485836:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575406.485838:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575406.485839:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575406.485841:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575406.485844:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575406.485844:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575406.485848:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575406.485850:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575406.485852:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575406.485853:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575406.485855:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000010:3.0:1788575406.485857:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000020:00000040:3.0:1788575406.485860:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575406.485862:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575406.485863:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1788575406.485864:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575406.485867:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575406.485868:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575406.485871:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1788575406.485871:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575406.485875:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000400:00000200:2.0:1788575406.485879:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575406.485879:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575406.485885:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575406.485886:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575406.485894:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330873:330873:256:4294967295] 192.168.202.110@tcp LPNI seq info [330830:330830:8:4294967295]
00000100:00000001:3.0:1788575406.485895:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575406.485898:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575406.485905:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575406.485906:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575406.485913:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575406.485915:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575406.485919:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575406.485924:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1788575406.485925:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12347)
00000800:00000200:2.0:1788575406.485933:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575406.485939:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575406.485945:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575406.485949:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788575406.485954:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575406.485957:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788575406.485961:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575406.485963:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575406.485969:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575406.485973:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575406.485977:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575406.485980:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575406.485983:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575406.485987:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.485995:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575406.485998:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486004:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575406.486006:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575406.486011:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575406.486014:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575406.486017:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307011001
00000100:00000010:2.0:1788575406.486022:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd80.
00000100:00000001:2.0:1788575406.486028:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575406.486033:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307011001
00000100:00000010:2.0:1788575406.486035:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c480.
00000100:00000001:2.0:1788575406.486038:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575406.486041:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575406.486044:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575406.486046:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575406.486050:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575406/real 1788575406]  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575406.486063:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575406.486064:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575406.486066:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575406.486067:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575406.486068:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575406.486070:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575406.486073:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575406.486089:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486093:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575406.486095:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486097:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575406.486098:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575406.486100:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486104:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575406.486107:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486111:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575406.486113:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486114:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575406.486115:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486116:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575406.486117:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575406.486119:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575406.486121:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575406.486122:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575406.486125:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575406.486130:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393207040:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575406.486133:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486134:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575406.486137:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393207040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486141:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575406.486143:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575406.486145:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575406.486147:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575406.486148:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575406.486149:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575406.486151:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575406.486152:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575406.486154:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575406.486156:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575406.486157:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575406.486159:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575406.486160:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486161:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486167:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575406.486169:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575406.486177:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575406.486182:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575406.486183:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575406.486187:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486194:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575406.486195:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575406.486197:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575406.486198:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575406.486201:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575406.486204:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575406.486207:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575406.486209:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575406.486211:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575406.486212:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575406.486215:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575406.486216:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575406.486219:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575406.486221:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575406.486223:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575406.486225:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393207168, portal 10
00000100:00000001:2.0:1788575406.486227:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575406.486227:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788575406.486232:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486236:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575406.486237:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575406.486239:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575406.486240:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393207168, offset 0
00000400:00000200:2.0:1788575406.486242:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575406.486245:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575406.486247:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575406.486249:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575406.486250:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575406.486252:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575406.486254:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575406.486256:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330874:330874:256:4294967295] 192.168.202.110@tcp LPNI seq info [330831:330831:8:4294967295]
00000400:00000200:2.0:1788575406.486258:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575406.486262:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575406.486266:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575406.486269:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575406.486272:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575406.486275:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788575406.486279:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575406.486281:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788575406.486285:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575406.486286:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575406.486289:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575406.486292:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575406.486295:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575406.486297:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575406.486299:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575406.486302:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486310:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575406.486313:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486319:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575406.486321:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575406.486324:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575406.486327:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575406.486330:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575406.486331:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307010086
00000100:00000001:2.0:1788575406.486334:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307011001
00000100:00000001:2.0:1788575406.486336:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307010086
00000100:00000001:2.0:1788575406.486338:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307011001
00000100:00000001:2.0:1788575406.486340:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575406.486342:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575406.486344:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575406.486348:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575406/real 1788575406]  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575406.486356:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575406.486357:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575406.486359:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575406.486361:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575406.486362:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575406.486364:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575406.486367:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575406.486375:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486380:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575406.486382:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486383:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575406.486384:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575406.486388:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486423:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575406.486426:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486431:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575406.486432:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486433:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575406.486434:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486435:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575406.486436:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575406.486438:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575406.486441:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575406.486443:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575406.486447:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575406.486455:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575406.486459:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575406.486462:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486468:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575406.486469:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575406.486471:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575406.486473:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575406.486475:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575406.486478:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575406.486481:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575406.486483:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575406.486484:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575406.486486:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575406.486489:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575406.486490:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575406.486493:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575406.486495:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575406.486497:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575406.486499:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393207296, portal 4
00000100:00000001:2.0:1788575406.486500:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575406.486501:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575406.486505:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486509:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575406.486510:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575406.486512:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575406.486513:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393207296, offset 0
00000400:00000200:2.0:1788575406.486515:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575406.486517:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575406.486520:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575406.486521:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575406.486523:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575406.486525:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575406.486526:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575406.486528:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330875:330875:256:4294967295] 192.168.202.110@tcp LPNI seq info [330832:330832:8:4294967295]
00000400:00000200:2.0:1788575406.486531:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575406.486535:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575406.486537:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575406.486539:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575406.486541:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575406.486542:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788575406.486545:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575406.486546:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788575406.486549:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575406.486550:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575406.486552:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575406.486554:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575406.486556:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575406.486557:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575406.486558:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575406.486560:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486565:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575406.486567:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486573:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575406.486574:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575406.486577:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575406.486579:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575406.486582:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575406.486584:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307010233
00000100:00000001:2.0:1788575406.486586:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307011001
00000100:00000001:2.0:1788575406.486588:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307010233
00000100:00000001:2.0:1788575406.486590:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307011001
00000100:00000001:2.0:1788575406.486591:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575406.486593:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575406.486595:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575406.486598:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575406/real 1788575406]  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575406.486605:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575406.486606:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575406.486607:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575406.486610:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575406.486611:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575406.486613:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575406.486616:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575406.486622:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486628:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575406.486629:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486630:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575406.486632:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575406.486635:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486640:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575406.486643:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486648:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575406.486650:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486651:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575406.486652:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486654:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575406.486656:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575406.486658:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575406.486661:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575406.486663:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575406.486666:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575406.486673:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575406.486676:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486677:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575406.486680:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393207168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486686:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575406.486687:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575406.486689:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575406.486691:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575406.486693:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575406.486694:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575406.486696:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575406.486697:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575406.486700:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575406.486701:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575406.486702:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575406.486704:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575406.486705:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486707:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575406.486709:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393207296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575406.486715:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575406.486716:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575406.486718:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575406.486720:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575406.486721:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575406.486722:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575406.486724:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575406.486725:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575406.486727:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575406.486728:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575406.486729:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575406.486731:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575406.486732:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486734:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575406.486737:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575406.486738:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575406.486740:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575406.486741:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575406.869480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575406.869487:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575406.869489:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575406.869492:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575406.869498:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575406.869500:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575406.869503:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575406.869505:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575406.869509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575406.869510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575406.869514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575406.869516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575406.869559:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575406.869563:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575406.869566:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575406.997513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575406.997530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575406.997532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575406.997537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575406.997541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575406.997545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575406.997547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575406.997550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575406.997553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575407.445512:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307011001, at 4307011008
00000100:00000010:0.0:1788575407.445525:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000100:00000040:1.0:1788575407.446135:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307011001, at 4307011008
00000100:00000010:1.0:1788575407.446142:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00010000:00000001:3.0:1788575407.893498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575407.893503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575407.893507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575407.893508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575407.893511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575407.893512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575407.893561:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575407.893564:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575407.893568:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575408.021484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575408.021491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575408.021493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575408.021498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575408.021501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575408.021505:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575408.021507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575408.021509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575408.021520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575408.917687:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575408.917693:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575408.917699:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575408.917700:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575408.917704:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575408.917705:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575408.918064:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575408.918067:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575408.918071:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575409.045547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575409.045553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575409.045555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575409.045559:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575409.045561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575409.045564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575409.045565:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575409.045567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575409.045569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575409.941530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575409.941536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575409.941540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575409.941541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575409.941544:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575409.941545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575409.941618:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575409.941623:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575409.941628:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575410.069497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575410.069502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575410.069504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575410.069507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575410.069510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575410.069512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575410.069514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575410.069516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575410.069518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575410.965482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575410.965487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575410.965490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575410.965491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575410.965493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575410.965494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575410.965556:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575410.965560:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575410.965563:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575411.093525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575411.093533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575411.093535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575411.093540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575411.093543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575411.093547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575411.093549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575411.093551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575411.093554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575411.605487:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575411.605495:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575411.605497:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575411.605500:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575411.605503:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575411.605508:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12342
00000400:00000200:1.0:1788575411.605514:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575411.605519:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575411.605521:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575411.605523:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605525:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575411.605527:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605529:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575411.605531:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575411.605533:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575411.605535:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575411.605537:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575411.605538:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575411.605540:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575411.605541:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605543:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575411.605544:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605545:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575411.605548:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575411.605550:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605552:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575411.605554:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575411.605556:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575411.605558:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575411.605559:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575411.605561:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575411.605569:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788575411.605571:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575411.605574:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575411.605575:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575411.605577:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575411.605579:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575411.605582:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575411.605587:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788575411.605591:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575411.605597:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575411.605604:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575411.605607:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575411.605611:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575411.605631:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575411.605634:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575411.605636:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575411.605637:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575411.605638:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575411.605640:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575411.605641:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12342
00000400:00000200:1.0:1788575411.605643:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575411.605645:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575411.605647:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575411.605648:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605650:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575411.605651:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605652:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575411.605653:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575411.605654:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575411.605655:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575411.605656:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575411.605657:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575411.605659:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575411.605660:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605662:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575411.605663:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605664:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575411.605665:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575411.605666:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605667:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575411.605668:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575411.605672:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575411.605678:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575411.605679:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575411.605682:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575411.605684:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575411.605685:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575411.605687:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788575411.605689:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575411.605691:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575411.605692:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575411.605693:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575411.605694:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575411.605696:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575411.605698:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788575411.605700:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575411.605703:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575411.605708:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575411.605709:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575411.605712:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575411.605721:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575411.605722:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1788575411.605726:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575411.605731:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575411.605732:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1788575411.605733:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1788575411.605734:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575411.605740:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:2.0:1788575411.605740:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00080000:1.0:1788575411.605747:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12342
00000400:00000200:1.0:1788575411.605752:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575411.605757:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788575411.605758:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575411.605765:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575411.605777:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1788575411.605782:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575411.605782:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575411.605788:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575411.605790:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:1.0:1788575411.605794:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575411.605796:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788575411.605800:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575411.605807:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575411.605808:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
02000000:00000001:2.0:1788575411.605808:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575411.605814:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575411.605818:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00100000:2.0:1788575411.605820:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393207424:192.168.202.110@tcp:250:kworker.0
00000020:00000001:1.0:1788575411.605822:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575411.605827:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575411.605833:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575411.605834:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1788575411.605836:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575411.605841:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605843:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575411.605845:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
02000000:00000001:2.0:1788575411.605845:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575411.605851:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575411.605852:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575411.605859:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575411.605859:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1788575411.605865:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575411.605869:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575411.605870:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575411.605871:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575411.605873:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
02000000:00000010:2.0:1788575411.605873:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
00000008:00000001:1.0:1788575411.605883:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575411.605883:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575411.605887:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575411.605891:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788575411.605893:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575411.605900:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575411.605900:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575411.605905:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575411.605911:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575411.605913:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1788575411.605915:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
02000000:00000001:1.0:1788575411.605916:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575411.605924:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788575411.605927:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575411.605931:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575411.605936:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575411.605939:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575411.605939:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000040:1.0:1788575411.605948:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575411.605954:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393207424, portal 25
00000100:00000001:1.0:1788575411.605955:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575411.605965:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575411.605965:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575411.605971:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
00000100:00000040:1.0:1788575411.605972:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575411.605977:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12352)
00000100:00000040:2.0:1788575411.605985:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.605994:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575411.605999:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575411.606002:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575411.606007:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393207424, offset 0
00000400:00000200:2.0:1788575411.606014:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575411.606023:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575411.606047:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575411.606053:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575411.606058:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575411.606062:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575411.606066:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575411.606069:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330876:330876:256:4294967295] 192.168.202.110@tcp LPNI seq info [330833:330833:8:4294967295]
00000400:00000200:2.0:1788575411.606074:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575411.606085:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575411.606092:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575411.606099:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575411.606106:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575411.606110:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788575411.606117:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575411.606120:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1788575411.606124:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575411.606128:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575411.606135:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575411.606140:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575411.606146:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575411.606149:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575411.606153:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575411.606158:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606169:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575411.606173:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606180:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575411.606182:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575411.606189:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575411.606193:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575411.606196:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307016121
00000100:00000010:2.0:1788575411.606203:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c0c0.
00000100:00000001:2.0:1788575411.606211:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575411.606218:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307016121
00000100:00000010:2.0:1788575411.606221:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ca20.
00000100:00000001:2.0:1788575411.606225:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575411.606229:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575411.606232:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575411.606234:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575411.606239:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575411/real 1788575411]  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575411.606249:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575411.606251:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575411.606253:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575411.606256:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575411.606258:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575411.606261:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575411.606265:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575411.606274:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606281:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575411.606283:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606285:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575411.606287:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575411.606291:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606298:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575411.606303:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606310:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575411.606313:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606315:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575411.606316:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606319:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575411.606322:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575411.606325:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575411.606330:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575411.606333:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575411.606337:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575411.606346:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393207424:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575411.606351:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575411.606354:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606367:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575411.606369:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575411.606372:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575411.606374:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575411.606376:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575411.606379:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207552:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575411.606381:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575411.606426:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575411.606427:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575411.606429:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575411.606432:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575411.606433:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575411.606438:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575411.606440:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575411.606444:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575411.606447:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393207552, portal 10
00000100:00000001:2.0:1788575411.606448:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575411.606450:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345473920 : -130483364077696 : ffff8953796f5f80)
00000100:00000040:2.0:1788575411.606456:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606463:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575411.606465:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575411.606468:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575411.606470:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393207552, offset 0
00000400:00000200:2.0:1788575411.606474:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575411.606478:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575411.606482:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575411.606485:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575411.606487:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575411.606488:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575411.606490:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575411.606492:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330877:330877:256:4294967295] 192.168.202.110@tcp LPNI seq info [330834:330834:8:4294967295]
00000400:00000200:2.0:1788575411.606495:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575411.606500:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575411.606503:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575411.606505:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575411.606507:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575411.606510:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788575411.606512:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575411.606514:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1788575411.606517:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575411.606518:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575411.606520:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575411.606522:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575411.606524:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575411.606525:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575411.606527:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575411.606529:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606535:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575411.606536:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606541:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575411.606542:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575411.606544:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575411.606547:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575411.606549:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575411.606550:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307015207
00000100:00000001:2.0:1788575411.606553:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307016121
00000100:00000001:2.0:1788575411.606554:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307015207
00000100:00000001:2.0:1788575411.606555:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307016121
00000100:00000001:2.0:1788575411.606557:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575411.606558:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575411.606560:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575411.606562:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575411/real 1788575411]  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575411.606567:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575411.606568:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575411.606568:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575411.606570:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575411.606571:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575411.606572:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575411.606574:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575411.606579:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606583:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575411.606584:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606585:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575411.606585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575411.606587:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606591:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575411.606593:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606597:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575411.606598:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606599:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575411.606600:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606601:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575411.606602:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575411.606604:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575411.606606:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575411.606607:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575411.606610:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575411.606614:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207552:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575411.606616:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606618:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575411.606619:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393207424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606623:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575411.606625:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575411.606627:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575411.606630:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575411.606631:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575411.606633:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575411.606634:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575411.606636:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:2.0:1788575411.606639:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575411.606641:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575411.606642:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:2.0:1788575411.606644:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575411.606645:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606646:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575411.606648:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5f80 x1875444393207552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606651:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575411.606652:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575411.606653:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575411.606654:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575411.606655:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575411.606655:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575411.606656:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575411.606658:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:2.0:1788575411.606659:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575411.606660:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575411.606661:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5f80.
00000100:00000001:2.0:1788575411.606663:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575411.606664:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606665:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606671:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575411.606673:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575411.606674:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575411.606675:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575411.606676:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575411.606678:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606682:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575411.606683:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575411.606685:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575411.606686:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575411.606687:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575411.606688:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207680:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575411.606690:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575411.606692:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575411.606693:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575411.606694:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575411.606696:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575411.606697:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575411.606699:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575411.606701:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575411.606703:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575411.606705:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393207680, portal 4
00000100:00000001:2.0:1788575411.606706:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575411.606707:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345465856 : -130483364085760 : ffff8953796f4000)
00000100:00000040:2.0:1788575411.606711:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606719:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575411.606720:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575411.606721:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575411.606723:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393207680, offset 0
00000400:00000200:2.0:1788575411.606725:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575411.606727:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575411.606729:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575411.606730:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575411.606732:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575411.606733:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575411.606735:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575411.606737:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330878:330878:256:4294967295] 192.168.202.110@tcp LPNI seq info [330835:330835:8:4294967295]
00000400:00000200:2.0:1788575411.606740:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575411.606743:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575411.606745:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575411.606747:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575411.606749:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575411.606751:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788575411.606753:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575411.606754:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1788575411.606757:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575411.606758:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575411.606760:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575411.606762:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575411.606764:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575411.606765:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575411.606766:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575411.606769:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606773:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575411.606775:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606783:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575411.606784:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575411.606785:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575411.606787:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575411.606789:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575411.606790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307015353
00000100:00000001:2.0:1788575411.606792:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307016121
00000100:00000001:2.0:1788575411.606793:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307015353
00000100:00000001:2.0:1788575411.606794:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307016121
00000100:00000001:2.0:1788575411.606795:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575411.606796:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575411.606797:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575411.606799:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575411/real 1788575411]  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575411.606803:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575411.606804:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575411.606805:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575411.606806:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575411.606807:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575411.606809:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575411.606811:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575411.606818:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606824:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575411.606825:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606827:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575411.606828:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575411.606832:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606839:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575411.606843:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606850:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575411.606852:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606854:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575411.606855:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606857:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575411.606858:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575411.606861:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575411.606863:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575411.606865:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575411.606867:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575411.606872:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207680:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575411.606875:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606876:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575411.606878:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4000 x1875444393207680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575411.606881:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575411.606882:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575411.606883:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575411.606884:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575411.606885:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575411.606886:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575411.606887:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575411.606888:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:2.0:1788575411.606889:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575411.606890:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575411.606891:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4000.
00000100:00000001:2.0:1788575411.606893:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575411.606894:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606895:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575411.606897:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575411.606898:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575411.606899:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575411.606900:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575411.989491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575411.989496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575411.989500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575411.989501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575411.989503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575411.989504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575411.989569:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575411.989575:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575411.989580:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575412.117454:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575412.117460:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575412.117461:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575412.117465:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575412.117468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575412.117470:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575412.117472:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575412.117474:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575412.117476:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575412.565482:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307016121, at 4307016128
00000100:00000040:1.0:1788575412.565482:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307016121, at 4307016128
00000100:00000010:1.0:1788575412.565496:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000100:00000010:0.0:1788575412.565498:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00010000:00000001:3.0:1788575413.013447:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575413.013451:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575413.013454:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575413.013455:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575413.013458:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575413.013459:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575413.013542:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575413.013548:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575413.013554:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575413.141533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575413.141540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575413.141542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575413.141547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575413.141551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575413.141555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575413.141557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575413.141560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575413.141562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575414.037469:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575414.037476:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575414.037481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575414.037485:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575414.037488:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575414.037491:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575414.037494:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575414.037497:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575414.037499:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575414.037501:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575414.037505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575414.037506:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575414.037507:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575414.037509:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575414.037510:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575414.037511:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575414.037512:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575414.037513:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575414.037566:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575414.037568:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575414.037571:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575414.165430:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575414.165436:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575414.165438:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575414.165442:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575414.165444:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575414.165447:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575414.165448:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575414.165450:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575414.165452:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575415.061495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575415.061502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575415.061507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575415.061508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575415.061511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575415.061512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575415.061566:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575415.061569:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575415.061572:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575415.189518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575415.189529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575415.189532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575415.189537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575415.189541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575415.189544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575415.189546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575415.189549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575415.189551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575416.085481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575416.085486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575416.085489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575416.085490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575416.085492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575416.085493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575416.085541:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575416.085544:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575416.085546:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575416.213474:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575416.213479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575416.213482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575416.213486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575416.213488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575416.213492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575416.213494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575416.213496:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575416.213498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575416.725488:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575416.725495:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575416.725497:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575416.725500:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575416.725503:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575416.725508:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12347
00000400:00000200:2.0:1788575416.725515:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575416.725520:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575416.725522:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575416.725524:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725526:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575416.725528:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725529:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575416.725532:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575416.725533:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575416.725535:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575416.725538:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575416.725539:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575416.725541:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575416.725542:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725543:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575416.725544:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725545:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575416.725548:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575416.725550:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725551:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575416.725556:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575416.725559:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575416.725561:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575416.725562:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575416.725563:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575416.725570:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02d80.
00000020:00000040:2.0:1788575416.725573:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575416.725575:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575416.725576:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575416.725578:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575416.725580:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575416.725584:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575416.725589:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788575416.725593:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575416.725600:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575416.725607:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575416.725610:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575416.725612:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009a9ccdb8] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575416.725632:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575416.725634:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575416.725636:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575416.725637:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575416.725638:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575416.725640:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575416.725641:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12347
00000400:00000200:2.0:1788575416.725643:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575416.725645:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575416.725647:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575416.725649:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725651:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575416.725653:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725654:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575416.725656:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575416.725657:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575416.725658:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575416.725659:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575416.725660:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575416.725662:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575416.725662:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1788575416.725667:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575416.725668:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575416.725669:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575416.725672:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575416.725673:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788575416.725674:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575416.725676:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575416.725680:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725682:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575416.725682:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575416.725684:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575416.725689:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575416.725693:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575416.725696:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575416.725700:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:2.0:1788575416.725701:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575416.725703:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788575416.725707:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575416.725709:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1788575416.725712:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575416.725713:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:2.0:1788575416.725718:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03b80.
00000020:00000040:2.0:1788575416.725722:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575416.725723:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575416.725727:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:3.0:1788575416.725730:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393207808:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1788575416.725731:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575416.725736:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1788575416.725738:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575416.725739:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575416.725742:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575416.725743:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575416.725746:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575416.725749:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575416.725750:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788575416.725756:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575416.725757:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575416.725760:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575416.725764:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575416.725769:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575416.725775:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000001:2.0:1788575416.725779:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575416.725783:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575416.725783:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000040:2.0:1788575416.725788:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037d167ca] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575416.725790:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393207808, portal 25
00000100:00000001:3.0:1788575416.725797:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575416.725799:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575416.725800:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00080000:2.0:1788575416.725802:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575416.725805:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575416.725808:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:3.0:1788575416.725809:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575416.725810:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575416.725814:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575416.725818:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12347
00000400:00000200:2.0:1788575416.725825:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575416.725828:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:2.0:1788575416.725829:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575416.725832:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575416.725836:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575416.725837:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725840:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575416.725841:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:3.0:1788575416.725841:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000001:2.0:1788575416.725846:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575416.725848:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000200:3.0:1788575416.725848:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393207808, offset 0
00000020:00000040:2.0:1788575416.725850:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575416.725852:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575416.725854:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575416.725856:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575416.725858:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575416.725860:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575416.725862:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725865:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575416.725866:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725867:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575416.725869:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575416.725872:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575416.725872:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575416.725877:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575416.725879:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575416.725881:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575416.725884:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575416.725884:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1788575416.725888:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575416.725891:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575416.725893:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575416.725894:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788575416.725895:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575416.725902:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00000.
00000400:00000200:3.0:1788575416.725902:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:2.0:1788575416.725907:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575416.725909:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:2.0:1788575416.725911:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575416.725915:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575416.725915:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725919:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575416.725923:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575416.725923:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330879:330879:256:4294967295] 192.168.202.110@tcp LPNI seq info [330836:330836:8:4294967295]
02000000:00000001:2.0:1788575416.725927:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575416.725933:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1788575416.725935:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788575416.725941:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575416.725950:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575416.725950:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575416.725963:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575416.725965:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575416.725969:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575416.725970:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00000040:2.0:1788575416.725975:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bf5936f] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788575416.725997:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575416.726002:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1788575416.726011:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575416.726015:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000100:00000001:2.0:1788575416.726017:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1788575416.726022:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575416.726023:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:3.0:1788575416.726024:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000100:00000040:2.0:1788575416.726030:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1788575416.726031:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575416.726036:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1788575416.726038:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12357)
00000400:00000200:3.0:1788575416.726041:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575416.726044:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575416.726047:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575416.726050:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726057:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575416.726059:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726064:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575416.726065:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575416.726070:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575416.726073:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575416.726075:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307021241
00000100:00000010:3.0:1788575416.726079:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575416.726084:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575416.726088:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307021241
00000100:00000010:3.0:1788575416.726090:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575416.726092:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575416.726094:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575416.726095:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575416.726097:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575416.726100:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575416/real 1788575416]  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575416.726106:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575416.726107:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575416.726109:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575416.726110:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575416.726111:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575416.726113:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575416.726115:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575416.726119:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726123:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575416.726124:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726126:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575416.726127:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575416.726129:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726133:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575416.726136:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726139:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575416.726141:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726142:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575416.726143:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726144:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575416.726146:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575416.726147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575416.726149:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575416.726150:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575416.726153:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575416.726157:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393207808:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575416.726162:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726163:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575416.726165:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393207808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726169:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575416.726170:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575416.726172:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575416.726174:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575416.726175:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575416.726176:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575416.726177:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575416.726179:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575416.726182:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575416.726183:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575416.726184:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:3.0:1788575416.726186:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575416.726187:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726188:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726193:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575416.726194:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575416.726196:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575416.726197:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575416.726197:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575416.726199:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726203:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575416.726204:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575416.726206:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575416.726207:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575416.726208:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575416.726210:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207936:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575416.726211:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575416.726212:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575416.726213:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575416.726215:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575416.726216:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575416.726218:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575416.726220:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575416.726221:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575416.726223:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575416.726225:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393207936, portal 10
00000100:00000001:3.0:1788575416.726227:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575416.726227:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:3.0:1788575416.726232:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726236:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575416.726237:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575416.726238:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575416.726240:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393207936, offset 0
00000400:00000200:3.0:1788575416.726241:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575416.726244:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575416.726246:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575416.726248:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575416.726249:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575416.726251:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575416.726252:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575416.726254:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330880:330880:256:4294967295] 192.168.202.110@tcp LPNI seq info [330837:330837:8:4294967295]
00000400:00000200:3.0:1788575416.726256:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575416.726260:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575416.726263:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575416.726264:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575416.726267:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575416.726268:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1788575416.726271:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575416.726273:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1788575416.726275:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575416.726276:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575416.726278:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575416.726280:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575416.726282:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575416.726283:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575416.726284:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575416.726286:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726290:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575416.726292:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726295:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575416.726296:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575416.726298:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575416.726300:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575416.726302:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575416.726303:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307020326
00000100:00000001:3.0:1788575416.726305:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307021241
00000100:00000001:3.0:1788575416.726306:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307020326
00000100:00000001:3.0:1788575416.726308:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307021241
00000100:00000001:3.0:1788575416.726309:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575416.726310:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575416.726311:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575416.726313:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575416/real 1788575416]  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575416.726317:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575416.726317:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575416.726318:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575416.726319:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575416.726320:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575416.726321:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575416.726323:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575416.726328:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726331:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575416.726332:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726333:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575416.726334:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575416.726336:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726339:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575416.726341:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726344:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575416.726346:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726346:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575416.726347:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726348:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575416.726349:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575416.726351:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575416.726352:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575416.726353:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575416.726355:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575416.726360:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393207936:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575416.726362:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575416.726364:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726371:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575416.726372:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575416.726373:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575416.726374:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575416.726375:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575416.726377:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575416.726378:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575416.726379:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575416.726380:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575416.726381:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575416.726383:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575416.726384:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575416.726386:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575416.726387:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575416.726389:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575416.726390:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393208064, portal 4
00000100:00000001:3.0:1788575416.726410:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575416.726411:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:3.0:1788575416.726414:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726418:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575416.726419:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575416.726421:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575416.726422:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393208064, offset 0
00000400:00000200:3.0:1788575416.726425:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575416.726427:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575416.726429:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575416.726430:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575416.726432:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575416.726433:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575416.726435:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575416.726437:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330881:330881:256:4294967295] 192.168.202.110@tcp LPNI seq info [330838:330838:8:4294967295]
00000400:00000200:3.0:1788575416.726445:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575416.726452:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575416.726454:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575416.726456:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575416.726458:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575416.726460:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1788575416.726462:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575416.726464:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1788575416.726466:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575416.726467:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575416.726469:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575416.726471:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575416.726473:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575416.726474:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575416.726475:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575416.726477:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726481:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575416.726483:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726487:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575416.726488:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575416.726489:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575416.726491:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575416.726493:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575416.726494:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307020473
00000100:00000001:3.0:1788575416.726496:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307021241
00000100:00000001:3.0:1788575416.726497:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307020473
00000100:00000001:3.0:1788575416.726498:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307021241
00000100:00000001:3.0:1788575416.726500:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575416.726501:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575416.726502:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575416.726504:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575416/real 1788575416]  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575416.726509:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575416.726509:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575416.726510:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575416.726511:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575416.726512:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575416.726513:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575416.726515:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575416.726520:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726523:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575416.726524:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726525:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575416.726525:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575416.726527:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726531:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575416.726533:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726536:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575416.726537:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726538:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575416.726539:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726540:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575416.726541:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575416.726542:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575416.726544:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575416.726545:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575416.726547:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575416.726552:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575416.726554:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726555:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575416.726556:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393207936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726560:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575416.726560:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575416.726561:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575416.726563:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575416.726563:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575416.726564:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575416.726566:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575416.726567:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575416.726568:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575416.726569:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575416.726570:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:3.0:1788575416.726571:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575416.726572:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726573:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575416.726574:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393208064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575416.726578:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575416.726578:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575416.726579:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575416.726580:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575416.726581:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575416.726581:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575416.726582:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575416.726584:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575416.726585:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575416.726585:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575416.726586:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:3.0:1788575416.726588:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575416.726588:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726589:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575416.726591:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575416.726592:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575416.726593:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575416.726593:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575417.109469:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575417.109475:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575417.109477:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575417.109478:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575417.109482:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575417.109484:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575417.109485:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575417.109486:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575417.109488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575417.109489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575417.109491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575417.109492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575417.109568:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575417.109571:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575417.109573:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575417.237483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575417.237488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575417.237490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575417.237494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575417.237497:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575417.237500:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575417.237502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575417.237504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575417.237506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575417.685523:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307021241, at 4307021248
00000100:00000040:1.0:1788575417.685523:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307021241, at 4307021248
00000100:00000010:1.0:1788575417.685538:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:0.0:1788575417.685541:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00010000:00000001:3.0:1788575418.133491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575418.133496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575418.133500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575418.133501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575418.133504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575418.133505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575418.133601:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575418.133605:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575418.133609:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575418.261488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575418.261495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575418.261497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575418.261502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575418.261505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575418.261508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575418.261511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575418.261512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575418.261515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575419.157485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575419.157490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575419.157494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575419.157495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575419.157497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575419.157498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575419.157556:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575419.157561:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575419.157565:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575419.285505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575419.285511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575419.285513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575419.285516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575419.285519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575419.285522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575419.285524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575419.285526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575419.285528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575420.181483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575420.181488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575420.181491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575420.181493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575420.181495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575420.181496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575420.181557:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575420.181563:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575420.181566:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575420.309480:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575420.309487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575420.309488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575420.309493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575420.309496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575420.309498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575420.309500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575420.309502:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575420.309504:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575421.205461:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575421.205468:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575421.205475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575421.205476:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575421.205479:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575421.205480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575421.205527:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575421.205531:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575421.205534:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575421.333509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575421.333516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575421.333518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575421.333522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575421.333525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575421.333528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575421.333530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575421.333532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575421.333534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575421.845501:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575421.845509:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575421.845511:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575421.845515:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575421.845519:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575421.845524:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12352
00000400:00000200:2.0:1788575421.845533:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575421.845538:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575421.845540:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575421.845543:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845546:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575421.845547:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845550:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575421.845553:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575421.845554:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575421.845557:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575421.845559:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575421.845561:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575421.845563:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575421.845564:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845566:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575421.845567:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845569:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575421.845571:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575421.845574:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845576:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575421.845578:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575421.845580:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575421.845582:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575421.845584:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575421.845585:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575421.845594:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00380.
00000020:00000040:2.0:1788575421.845596:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575421.845599:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575421.845600:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575421.845603:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575421.845605:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575421.845608:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575421.845614:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000100:00000001:2.0:1788575421.845618:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575421.845626:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575421.845638:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575421.845641:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575421.845644:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fc98b80] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575421.845662:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575421.845665:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575421.845667:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575421.845668:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575421.845670:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575421.845672:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575421.845674:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12352
00000400:00000200:2.0:1788575421.845677:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575421.845679:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575421.845681:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575421.845684:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845685:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575421.845688:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845690:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575421.845691:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575421.845692:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575421.845693:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575421.845695:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575421.845697:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575421.845697:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575421.845702:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575421.845702:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1788575421.845706:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575421.845706:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788575421.845709:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575421.845711:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845713:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575421.845716:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575421.845721:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:3.0:1788575421.845722:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575421.845724:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575421.845729:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575421.845735:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575421.845735:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1788575421.845739:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.845739:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575421.845743:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575421.845743:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575421.845747:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:2.0:1788575421.845749:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575421.845750:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788575421.845754:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575421.845757:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393208192:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575421.845758:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575421.845765:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01880.
00000100:00000001:3.0:1788575421.845765:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1788575421.845769:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575421.845770:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575421.845772:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575421.845774:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575421.845775:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575421.845777:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788575421.845778:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575421.845780:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575421.845782:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575421.845784:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000010:2.0:1788575421.845786:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575421.845787:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575421.845791:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575421.845796:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1788575421.845801:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575421.845801:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000001:2.0:1788575421.845810:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1788575421.845811:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000001:2.0:1788575421.845813:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575421.845817:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ab28ae01] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575421.845818:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393208192, portal 25
00000100:00000001:3.0:1788575421.845824:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575421.845826:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000001:2.0:1788575421.845829:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575421.845832:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575421.845834:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575421.845836:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575421.845839:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575421.845842:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575421.845846:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575421.845852:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12352
00000400:00000200:2.0:1788575421.845860:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575421.845863:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:2.0:1788575421.845864:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1788575421.845869:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575421.845872:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575421.845875:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393208192, offset 0
00000100:00080000:2.0:1788575421.845877:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575421.845882:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575421.845884:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845887:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575421.845888:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845890:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575421.845894:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1788575421.845895:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575421.845897:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575421.845899:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1788575421.845902:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:2.0:1788575421.845903:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575421.845906:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575421.845909:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575421.845909:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1788575421.845916:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575421.845917:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1788575421.845920:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575421.845923:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1788575421.845926:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575421.845930:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845931:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575421.845947:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330882:330882:256:4294967295] 192.168.202.110@tcp LPNI seq info [330839:330839:8:4294967295]
00000020:00000040:2.0:1788575421.845951:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575421.845957:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575421.845958:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575421.845962:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575421.845964:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575421.845967:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575421.845971:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575421.845972:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1788575421.845975:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575421.845979:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:3.0:1788575421.845979:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00000001:2.0:1788575421.845985:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575421.845986:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00000001:2.0:1788575421.845989:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575421.845995:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:2.0:1788575421.845997:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00000.
00000800:00000200:3.0:1788575421.845998:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000040:2.0:1788575421.846003:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:3.1:1788575421.846006:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1788575421.846010:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.1:1788575421.846011:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:2.0:1788575421.846015:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1788575421.846018:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575421.846020:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:3.0:1788575421.846021:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
02000000:00000001:2.0:1788575421.846025:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575421.846027:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1788575421.846030:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575421.846034:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000010:2.0:1788575421.846037:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000400:00000200:3.0:1788575421.846039:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000100:00000001:2.0:1788575421.846042:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575421.846042:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575421.846046:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00100000:2.0:1788575421.846050:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575421.846053:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575421.846072:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575421.846076:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.846077:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575421.846082:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575421.846087:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bf5936f] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846105:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575421.846109:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.846109:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:2.0:1788575421.846114:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575421.846116:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575421.846118:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1788575421.846122:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575421.846126:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12363)
00000100:00000001:3.0:1788575421.846126:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307026361
00000100:00000010:3.0:1788575421.846130:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788575421.846138:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575421.846144:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307026361
00000100:00000010:3.0:1788575421.846146:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575421.846148:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575421.846151:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575421.846153:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.846155:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575421.846158:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575421/real 1788575421]  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575421.846165:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575421.846166:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575421.846168:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575421.846170:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575421.846171:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575421.846172:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575421.846175:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575421.846180:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846185:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575421.846186:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846188:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575421.846189:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575421.846192:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846197:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575421.846200:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846205:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575421.846207:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846208:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575421.846209:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846211:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575421.846212:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575421.846214:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575421.846217:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575421.846218:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575421.846221:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575421.846226:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393208192:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575421.846229:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846230:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575421.846233:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393208192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846237:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575421.846240:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575421.846241:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575421.846244:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575421.846245:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575421.846246:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575421.846248:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575421.846249:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575421.846252:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575421.846254:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575421.846255:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:3.0:1788575421.846257:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575421.846259:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846260:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846266:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575421.846267:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.846269:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575421.846270:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575421.846271:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575421.846273:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846278:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575421.846279:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575421.846281:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575421.846283:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575421.846284:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575421.846286:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575421.846288:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575421.846289:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575421.846290:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575421.846291:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575421.846293:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575421.846294:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575421.846296:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575421.846298:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575421.846300:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575421.846302:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393208320, portal 10
00000100:00000001:3.0:1788575421.846304:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575421.846305:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530777728 : -130484178773888 : ffff895348e01880)
00000100:00000040:3.0:1788575421.846310:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846314:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575421.846316:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575421.846318:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575421.846319:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393208320, offset 0
00000400:00000200:3.0:1788575421.846322:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575421.846325:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575421.846327:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575421.846330:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575421.846331:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575421.846333:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575421.846335:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575421.846337:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330883:330883:256:4294967295] 192.168.202.110@tcp LPNI seq info [330840:330840:8:4294967295]
00000400:00000200:3.0:1788575421.846340:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575421.846344:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575421.846347:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575421.846349:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575421.846352:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575421.846354:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575421.846357:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575421.846359:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575421.846362:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575421.846363:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575421.846365:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575421.846367:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575421.846369:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575421.846371:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575421.846372:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575421.846375:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846380:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575421.846382:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846386:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.846387:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575421.846390:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575421.846412:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575421.846414:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575421.846416:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307025447
00000100:00000001:3.0:1788575421.846418:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307026361
00000100:00000001:3.0:1788575421.846420:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307025447
00000100:00000001:3.0:1788575421.846421:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307026361
00000100:00000001:3.0:1788575421.846423:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575421.846424:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.846425:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575421.846428:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575421/real 1788575421]  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575421.846433:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575421.846434:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575421.846435:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575421.846437:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575421.846438:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575421.846440:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575421.846442:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575421.846447:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846452:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575421.846452:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846454:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575421.846454:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575421.846457:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846460:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575421.846463:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846468:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575421.846470:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846471:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575421.846472:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846473:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575421.846474:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575421.846476:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575421.846478:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575421.846479:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575421.846481:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575421.846486:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575421.846488:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575421.846490:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846495:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575421.846496:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575421.846497:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575421.846498:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575421.846500:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575421.846501:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208448:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575421.846503:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575421.846504:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575421.846506:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575421.846507:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575421.846509:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575421.846511:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575421.846513:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575421.846516:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575421.846518:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575421.846520:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393208448, portal 4
00000100:00000001:3.0:1788575421.846521:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575421.846522:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:3.0:1788575421.846526:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846530:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575421.846532:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575421.846533:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575421.846534:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393208448, offset 0
00000400:00000200:3.0:1788575421.846537:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575421.846539:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575421.846541:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575421.846543:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575421.846545:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575421.846547:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575421.846548:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575421.846550:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330884:330884:256:4294967295] 192.168.202.110@tcp LPNI seq info [330841:330841:8:4294967295]
00000400:00000200:3.0:1788575421.846553:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575421.846558:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575421.846560:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575421.846562:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575421.846565:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575421.846567:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575421.846570:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575421.846572:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575421.846574:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575421.846576:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575421.846577:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575421.846580:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575421.846582:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575421.846584:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575421.846587:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575421.846591:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846599:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575421.846602:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846608:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.846619:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575421.846624:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575421.846627:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575421.846631:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575421.846633:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307025593
00000100:00000001:3.0:1788575421.846635:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307026361
00000100:00000001:3.0:1788575421.846638:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307025593
00000100:00000001:3.0:1788575421.846640:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307026361
00000100:00000001:3.0:1788575421.846642:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575421.846644:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.846646:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575421.846651:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575421/real 1788575421]  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575421.846659:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575421.846661:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575421.846663:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575421.846666:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575421.846668:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575421.846670:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575421.846674:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575421.846682:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846694:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575421.846695:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846697:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575421.846698:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575421.846700:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846705:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575421.846707:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846711:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575421.846713:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846714:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575421.846715:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846717:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575421.846718:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575421.846720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575421.846722:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575421.846723:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575421.846725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575421.846730:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208448:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575421.846733:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846734:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575421.846737:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01880 x1875444393208320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846741:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575421.846742:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575421.846744:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575421.846746:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575421.846747:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575421.846748:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575421.846750:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575421.846751:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575421.846753:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575421.846754:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575421.846755:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01880.
00000100:00000001:3.0:1788575421.846757:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575421.846758:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846759:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575421.846761:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393208448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575421.846765:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575421.846766:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575421.846766:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575421.846768:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575421.846768:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575421.846769:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575421.846770:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575421.846771:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575421.846772:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575421.846773:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575421.846774:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:3.0:1788575421.846775:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575421.846776:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846778:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575421.846780:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575421.846781:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575421.846782:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575421.846784:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575422.229510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575422.229516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575422.229520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575422.229521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575422.229523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575422.229525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575422.229597:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575422.229602:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575422.229606:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575422.357494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575422.357501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575422.357503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575422.357509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575422.357512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575422.357516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575422.357519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575422.357521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575422.357524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575422.805485:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307026361, at 4307026368
00000100:00000040:1.0:1788575422.805485:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307026361, at 4307026368
00000100:00000010:1.0:1788575422.805504:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000010:0.0:1788575422.805506:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00010000:00000001:3.0:1788575423.253506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575423.253513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575423.253519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575423.253520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575423.253524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575423.253525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575423.253590:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575423.253594:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575423.253598:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575423.381503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575423.381509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575423.381511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575423.381515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575423.381518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575423.381521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575423.381523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575423.381525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575423.381527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575424.277476:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575424.277481:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575424.277485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575424.277487:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575424.277490:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575424.277492:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575424.277495:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575424.277497:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575424.277498:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575424.277500:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575424.277502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575424.277503:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575424.277504:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575424.277505:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575424.277506:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575424.277507:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575424.277508:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575424.277510:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575424.277563:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575424.277567:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575424.277570:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575424.405555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575424.405564:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575424.405566:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575424.405572:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575424.405576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575424.405580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575424.405583:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575424.405586:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575424.405590:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575425.301510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575425.301515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575425.301520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575425.301521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575425.301523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575425.301524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575425.301574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575425.301577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575425.301580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575425.429496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575425.429503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575425.429505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575425.429511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575425.429515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575425.429519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575425.429521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575425.429524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575425.429527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575426.325478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575426.325484:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575426.325488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575426.325489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575426.325491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575426.325492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575426.325553:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575426.325558:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575426.325562:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575426.453547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575426.453554:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575426.453558:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575426.453564:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575426.453568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575426.453572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575426.453585:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575426.453588:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575426.453592:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575426.965508:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575426.965517:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575426.965520:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575426.965523:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575426.965527:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575426.965533:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12358
00000400:00000200:1.0:1788575426.965540:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575426.965551:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575426.965553:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575426.965556:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965558:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575426.965560:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965562:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575426.965565:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575426.965567:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575426.965569:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575426.965572:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575426.965574:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575426.965576:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575426.965578:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965579:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575426.965581:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965582:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575426.965584:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575426.965587:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965588:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575426.965591:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575426.965593:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575426.965595:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575426.965597:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575426.965599:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575426.965607:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788575426.965609:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575426.965612:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575426.965613:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575426.965616:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575426.965618:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575426.965621:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575426.965627:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788575426.965632:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575426.965639:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575426.965647:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575426.965651:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575426.965654:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575426.965670:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575426.965674:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575426.965678:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575426.965679:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575426.965681:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575426.965684:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575426.965688:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12358
00000400:00000200:1.0:1788575426.965691:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575426.965695:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575426.965697:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575426.965699:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788575426.965700:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965702:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575426.965704:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1788575426.965705:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575426.965706:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1788575426.965707:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575426.965709:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575426.965711:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575426.965713:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:3.0:1788575426.965713:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788575426.965715:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575426.965717:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575426.965720:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575426.965723:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575426.965723:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575426.965725:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575426.965725:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575426.965728:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965729:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575426.965729:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:1.0:1788575426.965732:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575426.965732:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1788575426.965735:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575426.965735:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575426.965738:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575426.965739:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393208576:192.168.202.110@tcp:250:kworker.0
02000000:00000001:1.0:1788575426.965740:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575426.965743:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788575426.965745:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575426.965747:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575426.965747:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1788575426.965748:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575426.965749:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575426.965751:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:1.0:1788575426.965753:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575426.965756:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575426.965757:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
00000100:00000001:1.0:1788575426.965759:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575426.965759:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575426.965762:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788575426.965764:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:3.0:1788575426.965765:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575426.965767:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575426.965768:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575426.965769:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000001:1.0:1788575426.965770:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575426.965771:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575426.965774:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575426.965776:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
02000000:00000010:1.0:1788575426.965777:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788575426.965780:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575426.965783:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393208576, portal 25
00000100:00100000:1.0:1788575426.965785:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575426.965786:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575426.965787:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345466752 : -130483364084864 : ffff8953796f4380)
00000100:00000001:1.0:1788575426.965793:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1788575426.965793:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575426.965796:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575426.965800:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.965802:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575426.965805:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575426.965808:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000001:1.0:1788575426.965809:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575426.965812:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1788575426.965812:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393208576, offset 0
00000100:00080000:1.0:1788575426.965815:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575426.965816:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575426.965818:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575426.965819:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965821:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575426.965824:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12358
00000400:00000200:1.0:1788575426.965827:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575426.965828:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575426.965830:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575426.965833:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575426.965834:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:1.0:1788575426.965835:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965838:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575426.965840:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575426.965840:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1788575426.965842:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575426.965844:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:1.0:1788575426.965845:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575426.965846:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575426.965848:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1788575426.965848:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000040:1.0:1788575426.965850:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575426.965851:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:1.0:1788575426.965852:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575426.965855:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575426.965856:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330885:330885:256:4294967295] 192.168.202.110@tcp LPNI seq info [330842:330842:8:4294967295]
00000100:00000040:1.0:1788575426.965858:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965860:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575426.965861:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788575426.965862:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965864:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575426.965866:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575426.965868:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575426.965871:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575426.965878:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965880:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575426.965884:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
02000000:00000001:1.0:1788575426.965899:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575426.965903:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575426.965905:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575426.965906:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788575426.965907:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575426.965909:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:3.0:1788575426.965911:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000008:00000001:1.0:1788575426.965912:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575426.965914:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575426.965916:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:1.0:1788575426.965918:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000800:00000200:3.1:1788575426.965920:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000040:1.0:1788575426.965921:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575426.965924:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575426.965925:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1788575426.965925:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575426.965927:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:3.0:1788575426.965928:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
02000000:00000001:1.0:1788575426.965929:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575426.965930:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575426.965934:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:1.0:1788575426.965935:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788575426.965938:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575426.965939:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00100000:1.0:1788575426.965943:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575426.965946:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575426.965948:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575426.965953:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:1.0:1788575426.965957:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1788575426.965957:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575426.965959:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575426.965963:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.965967:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1788575426.965971:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575426.965971:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575426.965974:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575426.965979:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575426.965979:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575426.965981:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:1.0:1788575426.965984:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12368)
00000100:00000001:3.0:1788575426.965989:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575426.966001:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575426.966005:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307031481
00000100:00000010:3.0:1788575426.966011:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575426.966017:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575426.966023:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307031481
00000100:00000010:3.0:1788575426.966027:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788575426.966030:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575426.966032:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575426.966034:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575426.966036:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575426.966040:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575426/real 1788575426]  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575426.966048:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575426.966049:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575426.966051:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575426.966053:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575426.966054:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575426.966056:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575426.966058:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575426.966064:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966068:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575426.966070:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966071:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575426.966080:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575426.966083:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966087:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575426.966090:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966095:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575426.966097:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966099:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575426.966100:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966101:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575426.966103:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575426.966105:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575426.966108:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575426.966109:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575426.966112:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575426.966117:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393208576:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575426.966120:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966122:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575426.966124:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4380 x1875444393208576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966128:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575426.966130:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575426.966132:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575426.966134:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575426.966136:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575426.966137:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575426.966139:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575426.966141:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:3.0:1788575426.966143:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575426.966145:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575426.966146:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4380.
00000100:00000001:3.0:1788575426.966148:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575426.966149:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966151:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966157:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575426.966158:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575426.966160:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575426.966161:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575426.966162:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575426.966164:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966173:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575426.966174:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575426.966175:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575426.966177:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575426.966178:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575426.966180:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208704:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575426.966182:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575426.966183:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575426.966184:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575426.966185:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575426.966187:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575426.966189:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575426.966191:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575426.966194:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575426.966196:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575426.966198:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393208704, portal 10
00000100:00000001:3.0:1788575426.966199:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575426.966200:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345468544 : -130483364083072 : ffff8953796f4a80)
00000100:00000040:3.0:1788575426.966205:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966210:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575426.966211:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575426.966213:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575426.966214:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393208704, offset 0
00000400:00000200:3.0:1788575426.966217:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575426.966220:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575426.966222:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575426.966224:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575426.966226:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575426.966228:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575426.966230:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575426.966232:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330886:330886:256:4294967295] 192.168.202.110@tcp LPNI seq info [330843:330843:8:4294967295]
00000400:00000200:3.0:1788575426.966235:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575426.966240:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575426.966243:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575426.966245:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575426.966248:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575426.966250:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788575426.966253:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575426.966255:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575426.966257:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575426.966259:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575426.966261:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575426.966263:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575426.966265:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575426.966266:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575426.966268:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575426.966270:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966275:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575426.966277:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966282:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575426.966283:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575426.966285:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575426.966288:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575426.966290:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575426.966292:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307030566
00000100:00000001:3.0:1788575426.966294:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307031481
00000100:00000001:3.0:1788575426.966295:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307030566
00000100:00000001:3.0:1788575426.966297:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307031481
00000100:00000001:3.0:1788575426.966298:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575426.966300:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575426.966301:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575426.966304:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575426/real 1788575426]  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575426.966309:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575426.966310:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575426.966311:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575426.966312:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575426.966313:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575426.966315:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575426.966316:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575426.966324:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966331:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575426.966332:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966334:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575426.966336:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575426.966340:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966347:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575426.966352:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966359:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575426.966362:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966364:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575426.966365:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966367:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575426.966369:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575426.966372:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575426.966375:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575426.966378:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575426.966382:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575426.966388:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208704:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575426.966408:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575426.966410:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966415:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575426.966417:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575426.966418:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575426.966419:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575426.966421:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575426.966423:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208832:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575426.966425:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575426.966426:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575426.966427:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575426.966428:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575426.966431:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575426.966433:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575426.966435:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575426.966437:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575426.966440:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575426.966442:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393208832, portal 4
00000100:00000001:3.0:1788575426.966443:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575426.966444:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345476608 : -130483364075008 : ffff8953796f6a00)
00000100:00000040:3.0:1788575426.966449:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966453:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575426.966455:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575426.966456:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575426.966458:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393208832, offset 0
00000400:00000200:3.0:1788575426.966460:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575426.966463:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575426.966466:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575426.966468:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575426.966470:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575426.966471:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575426.966473:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575426.966475:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330887:330887:256:4294967295] 192.168.202.110@tcp LPNI seq info [330844:330844:8:4294967295]
00000400:00000200:3.0:1788575426.966478:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575426.966482:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575426.966485:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575426.966487:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575426.966489:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575426.966491:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788575426.966494:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575426.966495:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575426.966498:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575426.966500:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575426.966502:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575426.966504:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575426.966506:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575426.966507:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575426.966509:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575426.966512:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966517:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575426.966519:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966523:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575426.966524:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575426.966526:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575426.966528:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575426.966530:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575426.966532:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307030713
00000100:00000001:3.0:1788575426.966533:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307031481
00000100:00000001:3.0:1788575426.966535:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307030713
00000100:00000001:3.0:1788575426.966537:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307031481
00000100:00000001:3.0:1788575426.966538:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575426.966540:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575426.966541:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575426.966543:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575426/real 1788575426]  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575426.966548:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575426.966549:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575426.966550:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575426.966551:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575426.966552:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575426.966553:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575426.966555:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575426.966560:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966564:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575426.966565:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966566:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575426.966567:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575426.966569:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966573:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575426.966576:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966581:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575426.966583:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966584:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575426.966585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966586:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575426.966587:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575426.966589:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575426.966590:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575426.966592:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575426.966594:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575426.966599:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393208832:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575426.966601:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966602:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575426.966604:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4a80 x1875444393208704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966608:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575426.966609:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575426.966610:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575426.966612:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575426.966613:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575426.966614:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575426.966616:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575426.966617:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:3.0:1788575426.966619:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575426.966620:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575426.966621:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4a80.
00000100:00000001:3.0:1788575426.966623:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575426.966624:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966625:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575426.966627:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6a00 x1875444393208832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575426.966632:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575426.966633:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575426.966633:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575426.966635:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575426.966635:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575426.966636:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575426.966637:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575426.966638:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:3.0:1788575426.966639:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575426.966640:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575426.966641:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6a00.
00000100:00000001:3.0:1788575426.966642:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575426.966643:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966644:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575426.966646:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575426.966647:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575426.966649:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575426.966650:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575427.349541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575427.349551:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575427.349553:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575427.349557:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575427.349561:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575427.349564:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575427.349566:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575427.349569:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575427.349573:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575427.349575:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575427.349579:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575427.349580:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575427.349639:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575427.349644:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575427.349648:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575427.477487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575427.477495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575427.477498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575427.477505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575427.477510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575427.477515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575427.477518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575427.477522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575427.477526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575427.925473:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307031481, at 4307031488
00000100:00000040:1.0:1788575427.925474:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307031481, at 4307031488
00000100:00000010:0.0:1788575427.925489:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788575427.925491:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00010000:00000001:3.0:1788575428.373477:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575428.373484:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575428.373489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575428.373489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575428.373493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575428.373494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575428.373561:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575428.373566:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575428.373571:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575428.501521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575428.501528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575428.501529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575428.501533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575428.501536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575428.501539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575428.501541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575428.501543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575428.501546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575429.397510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575429.397515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575429.397520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575429.397521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575429.397523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575429.397524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575429.397573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575429.397576:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575429.397580:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575429.525473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575429.525480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575429.525483:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575429.525488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575429.525492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575429.525495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575429.525498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575429.525500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575429.525502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575430.421495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575430.421500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575430.421504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575430.421504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575430.421507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575430.421508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575430.421566:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575430.421570:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575430.421572:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575430.549514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575430.549519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575430.549521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575430.549525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575430.549528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575430.549531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575430.549534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575430.549536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575430.549545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575431.445522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575431.445527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575431.445531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575431.445532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575431.445535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575431.445536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575431.445609:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575431.445614:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575431.445619:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575431.573471:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575431.573479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575431.573481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575431.573486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575431.573490:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575431.573494:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575431.573497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575431.573499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575431.573503:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575432.085518:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575432.085527:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575432.085530:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575432.085533:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575432.085537:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575432.085543:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12363
00000400:00000200:3.0:1788575432.085551:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575432.085556:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575432.085558:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575432.085561:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085563:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575432.085565:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085567:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575432.085570:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575432.085572:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575432.085574:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575432.085577:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575432.085586:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575432.085588:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575432.085590:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085592:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575432.085593:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085595:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575432.085597:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575432.085600:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085602:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575432.085604:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575432.085606:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575432.085608:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.085609:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575432.085611:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575432.085621:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0a80.
00000020:00000040:3.0:1788575432.085625:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575432.085628:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575432.085630:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.085633:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575432.085636:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575432.085640:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575432.085648:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa000.
00000100:00000001:3.0:1788575432.085655:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575432.085665:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575432.085676:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575432.085679:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575432.085683:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b298c26f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575432.085708:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575432.085710:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575432.085713:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575432.085714:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575432.085715:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575432.085717:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575432.085719:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12363
00000400:00000200:3.0:1788575432.085722:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575432.085724:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575432.085726:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575432.085728:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085730:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575432.085732:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085734:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575432.085736:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575432.085737:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575432.085738:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575432.085740:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575432.085741:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575432.085743:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575432.085745:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085748:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575432.085749:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085751:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575432.085753:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1788575432.085753:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575432.085758:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575432.085760:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575432.085760:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575432.085765:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575432.085766:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1788575432.085770:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575432.085774:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.085776:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1788575432.085779:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1788575432.085780:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575432.085784:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.085788:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.085797:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575432.085802:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575432.085802:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2300.
00000020:00000040:3.0:1788575432.085808:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575432.085809:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575432.085810:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575432.085814:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.085815:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575432.085816:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575432.085818:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575432.085820:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:3.0:1788575432.085821:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
02000000:00000001:2.0:1788575432.085829:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.085829:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575432.085851:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:2.0:1788575432.085856:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393208960:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575432.085859:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575432.085864:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.085867:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:3.0:1788575432.085869:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc4bce75] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575432.085874:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575432.085878:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.085881:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575432.085882:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:3.0:1788575432.085886:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575432.085889:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
02000000:00000010:2.0:1788575432.085891:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
00000100:00000001:3.0:1788575432.085891:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575432.085894:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:2.0:1788575432.085898:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.085899:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575432.085903:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12363
00000400:00000200:3.0:1788575432.085907:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575432.085911:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575432.085911:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575432.085917:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575432.085920:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:2.0:1788575432.085922:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000001:3.0:1788575432.085922:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575432.085927:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085930:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575432.085931:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575432.085932:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000010:2.0:1788575432.085934:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000020:00000001:3.0:1788575432.085934:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575432.085935:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575432.085941:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575432.085942:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575432.085945:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575432.085946:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393208960, portal 25
00000100:00000001:3.0:1788575432.085947:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575432.085950:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575432.085954:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575432.085955:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575432.085960:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000020:00000040:3.0:1788575432.085960:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575432.085964:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575432.085968:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575432.085970:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575432.085973:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575432.085975:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575432.085977:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.085983:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575432.085987:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575432.085993:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.085995:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575432.085997:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575432.086002:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000010:3.0:1788575432.086005:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000400:00000010:2.0:1788575432.086008:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000020:00000040:3.0:1788575432.086011:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000200:2.0:1788575432.086015:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393208960, offset 0
02000000:00000001:3.0:1788575432.086017:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575432.086020:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575432.086024:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575432.086025:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575432.086029:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575432.086030:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575432.086036:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788575432.086043:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575432.086044:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575432.086056:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:3.0:1788575432.086061:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575432.086063:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575432.086070:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575432.086078:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575432.086082:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575432.086084:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575432.086088:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575432.086093:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330888:330888:256:4294967295] 192.168.202.110@tcp LPNI seq info [330845:330845:8:4294967295]
00000100:00000040:3.0:1788575432.086096:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575432.086107:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575432.086117:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575432.086122:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575432.086123:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1788575432.086128:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:2.0:1788575432.086137:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575432.086137:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12373)
00000800:00000010:2.0:1788575432.086148:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575432.086158:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575432.086162:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575432.086169:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575432.086172:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575432.086177:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575432.086179:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575432.086186:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575432.086191:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575432.086197:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575432.086201:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575432.086205:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575432.086209:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086219:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575432.086223:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575457 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086237:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.086240:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575432.086245:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575432.086250:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575432.086253:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307036601
00000100:00000010:2.0:1788575432.086258:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c420.
00000100:00000001:2.0:1788575432.086265:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575432.086271:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307036601
00000100:00000010:2.0:1788575432.086275:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c9c0.
00000100:00000001:2.0:1788575432.086278:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575432.086282:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575432.086285:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.086287:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575432.086293:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575432/real 1788575432]  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575432.086302:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575432.086304:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575432.086306:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575432.086309:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575432.086311:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575432.086313:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575432.086316:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575432.086325:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086332:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575432.086334:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.086336:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575432.086338:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575432.086343:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086351:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575432.086356:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086364:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575432.086367:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.086369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575432.086370:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.086373:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575432.086375:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575432.086378:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575432.086382:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575432.086384:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575432.086389:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575432.086431:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393208960:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575432.086436:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.086438:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575432.086442:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393208960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086449:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575432.086451:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575432.086454:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575432.086458:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575432.086460:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575432.086461:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575432.086464:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575432.086466:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:2.0:1788575432.086471:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575432.086473:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575432.086475:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:2.0:1788575432.086478:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575432.086479:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.086482:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.086489:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575432.086491:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.086493:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575432.086495:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575432.086496:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575432.086500:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086508:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575432.086510:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575432.086512:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575432.086514:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575432.086517:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575432.086520:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575432.086524:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575432.086526:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575432.086528:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575432.086530:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575432.086533:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575432.086535:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575432.086540:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575432.086542:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575432.086546:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575432.086549:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393209088, portal 10
00000100:00000001:2.0:1788575432.086551:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575432.086552:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000040:2.0:1788575432.086559:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086566:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575432.086568:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575432.086571:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575432.086574:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393209088, offset 0
00000400:00000200:2.0:1788575432.086579:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575432.086583:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575432.086587:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575432.086590:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575432.086593:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575432.086596:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575432.086598:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575432.086601:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330889:330889:256:4294967295] 192.168.202.110@tcp LPNI seq info [330846:330846:8:4294967295]
00000400:00000200:2.0:1788575432.086606:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575432.086613:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575432.086618:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575432.086621:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575432.086625:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575432.086628:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575432.086633:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575432.086636:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575432.086640:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575432.086642:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575432.086645:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575432.086649:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575432.086652:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575432.086654:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575432.086657:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575432.086661:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086669:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575432.086672:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575457 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086681:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.086682:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575432.086686:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575432.086690:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575432.086693:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575432.086696:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307035687
00000100:00000001:2.0:1788575432.086699:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307036601
00000100:00000001:2.0:1788575432.086701:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307035687
00000100:00000001:2.0:1788575432.086704:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307036601
00000100:00000001:2.0:1788575432.086705:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575432.086708:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.086710:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575432.086714:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575432/real 1788575432]  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575432.086723:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575432.086724:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575432.086726:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575432.086728:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575432.086730:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575432.086733:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575432.086736:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575432.086745:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086752:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575432.086754:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.086755:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575432.086757:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575432.086761:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086768:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575432.086773:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086780:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575432.086782:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.086784:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575432.086785:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.086788:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575432.086790:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575432.086793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575432.086796:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575432.086799:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575432.086803:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575432.086812:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575432.086816:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575432.086820:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086828:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575432.086829:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575432.086832:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575432.086834:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575432.086836:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575432.086856:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575432.086860:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575432.086862:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575432.086864:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575432.086866:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575432.086870:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575432.086872:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575432.086876:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575432.086880:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575432.086884:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575432.086887:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393209216, portal 4
00000100:00000001:2.0:1788575432.086888:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575432.086890:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575432.086897:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.086904:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575432.086906:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575432.086909:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575432.086912:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393209216, offset 0
00000400:00000200:2.0:1788575432.086916:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575432.086920:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575432.086924:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575432.086927:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575432.086930:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575432.086933:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575432.086935:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575432.086938:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330890:330890:256:4294967295] 192.168.202.110@tcp LPNI seq info [330847:330847:8:4294967295]
00000400:00000200:2.0:1788575432.086943:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575432.086950:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575432.086955:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575432.086959:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575432.086963:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575432.086966:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575432.086970:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575432.086974:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575432.086978:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575432.086980:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575432.086983:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575432.086987:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575432.086990:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575432.086993:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575432.086996:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575432.087000:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.087008:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575432.087011:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575457 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.087018:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.087020:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575432.087023:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575432.087026:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575432.087030:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575432.087032:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307035833
00000100:00000001:2.0:1788575432.087035:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307036601
00000100:00000001:2.0:1788575432.087037:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307035833
00000100:00000001:2.0:1788575432.087040:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307036601
00000100:00000001:2.0:1788575432.087042:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575432.087044:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.087046:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575432.087050:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575432/real 1788575432]  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575432.087059:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575432.087061:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575432.087063:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575432.087065:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575432.087067:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575432.087069:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575432.087073:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575432.087082:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.087096:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575432.087098:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.087100:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575432.087101:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575432.087105:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.087113:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575432.087118:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.087125:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575432.087128:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.087129:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575432.087130:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.087133:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575432.087135:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575432.087138:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575432.087141:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575432.087143:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575432.087147:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575432.087156:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575432.087160:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.087162:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575432.087165:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393209088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.087173:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575432.087174:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575432.087177:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575432.087180:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575432.087182:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575432.087184:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575432.087186:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575432.087187:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:2.0:1788575432.087190:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575432.087191:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575432.087193:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:2.0:1788575432.087196:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575432.087197:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.087199:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575432.087203:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393209216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575457 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575432.087210:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575432.087211:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575432.087213:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575432.087215:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575432.087216:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575432.087218:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575432.087220:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575432.087221:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575432.087224:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575432.087225:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575432.087227:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575432.087229:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575432.087231:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.087233:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.087237:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575432.087238:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575432.087240:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575432.087242:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575432.469508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575432.469513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575432.469517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575432.469518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575432.469521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575432.469522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575432.469582:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575432.469585:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575432.469594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575432.597510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575432.597516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575432.597518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575432.597523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575432.597526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575432.597529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.597532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575432.597534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575432.597536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575433.045487:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307036601, at 4307036608
00000100:00000040:1.0:1788575433.045487:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307036601, at 4307036608
00000100:00000010:1.0:1788575433.045503:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000100:00000010:0.0:1788575433.045507:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00010000:00000001:3.0:1788575433.493467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575433.493474:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575433.493481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575433.493482:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575433.493486:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575433.493487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575433.493614:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575433.493617:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575433.493620:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575433.621543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575433.621559:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575433.621561:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575433.621567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575433.621571:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575433.621575:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575433.621578:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575433.621581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575433.621584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575434.517480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575434.517487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575434.517492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575434.517495:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575434.517497:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575434.517500:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575434.517503:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575434.517505:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575434.517506:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575434.517508:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575434.517511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575434.517512:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575434.517513:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575434.517515:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575434.517516:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575434.517517:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575434.517518:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575434.517519:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575434.517615:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575434.517622:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575434.517626:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575434.645539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575434.645545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575434.645548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575434.645553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575434.645557:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575434.645563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575434.645567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575434.645571:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575434.645574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575435.541493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575435.541498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575435.541503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575435.541504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575435.541508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575435.541509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575435.541586:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575435.541591:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575435.541594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575435.669488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575435.669494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575435.669496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575435.669501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575435.669503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575435.669507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575435.669509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575435.669511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575435.669513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575436.565484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575436.565490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575436.565493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575436.565494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575436.565497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575436.565498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575436.565563:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575436.565567:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575436.565570:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575436.693500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575436.693505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575436.693507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575436.693512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575436.693515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575436.693517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575436.693519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575436.693522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575436.693524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575437.205499:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575437.205510:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575437.205513:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575437.205518:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575437.205523:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575437.205530:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12368
00000400:00000200:2.0:1788575437.205538:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575437.205547:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575437.205550:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575437.205554:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205558:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575437.205562:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205565:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575437.205570:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575437.205573:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575437.205577:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575437.205581:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575437.205584:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575437.205587:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575437.205590:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205592:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575437.205594:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205596:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575437.205598:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575437.205601:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205603:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575437.205605:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575437.205608:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575437.205610:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575437.205611:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575437.205613:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575437.205620:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03b80.
00000020:00000040:2.0:1788575437.205623:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575437.205626:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575437.205627:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575437.205629:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575437.205631:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575437.205634:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575437.205640:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000100:00000001:2.0:1788575437.205645:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575437.205652:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575437.205660:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575437.205663:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575437.205667:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037d167ca] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575437.205685:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575437.205687:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575437.205690:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575437.205691:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575437.205692:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575437.205694:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575437.205696:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12368
00000400:00000200:2.0:1788575437.205700:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575437.205702:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575437.205704:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575437.205707:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205708:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575437.205710:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205712:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575437.205713:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575437.205714:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575437.205716:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575437.205716:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1788575437.205719:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575437.205722:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1788575437.205723:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575437.205727:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788575437.205728:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575437.205737:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575437.205739:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575437.205740:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575437.205745:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205751:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575437.205758:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575437.205759:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575437.205764:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575437.205765:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205771:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575437.205771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575437.205776:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575437.205777:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575437.205782:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575437.205785:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1788575437.205788:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393209344:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1788575437.205805:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.205808:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1788575437.205810:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575437.205815:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:2.0:1788575437.205816:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575437.205819:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788575437.205822:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575437.205824:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788575437.205827:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575437.205832:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575437.205836:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575437.205841:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02d80.
00000400:00000200:3.0:1788575437.205843:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575437.205846:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575437.205849:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:3.0:1788575437.205850:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
02000000:00000001:2.0:1788575437.205851:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575437.205856:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575437.205858:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575437.205859:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575437.205862:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
02000000:00000010:2.0:1788575437.205865:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11400.
00000100:00000001:2.0:1788575437.205870:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575437.205872:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393209344, portal 25
00000100:00100000:2.0:1788575437.205876:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575437.205878:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575437.205881:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000001:2.0:1788575437.205891:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1788575437.205893:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575437.205896:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575437.205907:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009a9ccdb8] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.205909:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575437.205916:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575437.205921:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000001:2.0:1788575437.205928:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575437.205928:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393209344, offset 0
00000100:00080000:2.0:1788575437.205935:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575437.205939:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575437.205942:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575437.205944:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575437.205944:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575437.205950:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575437.205953:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12368
00000400:00000200:3.0:1788575437.205957:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575437.205966:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575437.205972:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575437.205976:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575437.205980:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575437.205983:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575437.205988:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330891:330891:256:4294967295] 192.168.202.110@tcp LPNI seq info [330848:330848:8:4294967295]
00000400:00000200:3.0:1788575437.205993:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575437.206000:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575437.206011:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1788575437.206012:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1788575437.206019:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1788575437.206023:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575437.206028:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000010:3.0:1788575437.206030:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000100:00000001:2.0:1788575437.206032:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575437.206038:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575437.206040:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575437.206043:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1788575437.206045:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575437.206051:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:3.1:1788575437.206053:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1788575437.206054:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575437.206058:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.1:1788575437.206060:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1788575437.206066:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575437.206069:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000020:00000040:2.0:1788575437.206070:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575437.206076:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575437.206076:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:2.0:1788575437.206082:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575437.206086:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1788575437.206091:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575437.206096:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575437.206096:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000100:00000040:2.0:1788575437.206101:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:3.0:1788575437.206102:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:2.0:1788575437.206106:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575437.206110:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:3.0:1788575437.206110:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00080000:2.0:1788575437.206116:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575437.206120:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575437.206121:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575437.206122:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575437.206126:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575437.206130:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575437.206135:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575437.206138:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575437.206142:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575437.206144:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206145:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000010:2.0:1788575437.206149:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03100.
00000100:00000040:3.0:1788575437.206151:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575462 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788575437.206154:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575437.206161:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575437.206163:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206168:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575437.206169:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1788575437.206172:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:2.0:1788575437.206173:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575437.206177:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206181:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575437.206183:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000400:00000200:3.0:1788575437.206188:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575437.206192:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307041721
00000100:00000001:2.0:1788575437.206198:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575437.206199:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00100000:2.0:1788575437.206206:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575437.206208:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575437.206217:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575437.206219:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575437.206222:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008692924c] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206226:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307041721
00000100:00000001:2.0:1788575437.206229:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575437.206233:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000040:2.0:1788575437.206234:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575437.206241:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:2.0:1788575437.206244:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575437.206248:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575437.206251:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12378)
00000100:00000001:3.0:1788575437.206254:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206258:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575437.206266:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575437/real 1788575437]  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575437.206278:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575437.206280:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575437.206282:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575437.206286:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575437.206287:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575437.206290:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575437.206294:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575437.206303:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206311:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575437.206313:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206315:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575437.206316:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575437.206320:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206324:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575437.206327:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206332:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575437.206334:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206335:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575437.206336:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206338:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575437.206339:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575437.206341:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575437.206344:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575437.206345:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575437.206348:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575437.206353:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393209344:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575437.206356:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206358:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575437.206360:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393209344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206364:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575437.206366:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575437.206369:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575437.206371:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575437.206373:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575437.206374:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575437.206376:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575437.206377:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575437.206380:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575437.206382:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575437.206383:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:3.0:1788575437.206386:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575437.206386:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206388:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206411:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575437.206412:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206414:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575437.206415:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575437.206416:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575437.206419:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206424:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575437.206425:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575437.206427:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575437.206428:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575437.206430:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575437.206432:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209472:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575437.206434:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575437.206436:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575437.206437:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575437.206438:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575437.206440:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575437.206442:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575437.206444:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575437.206446:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575437.206449:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575437.206452:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393209472, portal 10
00000100:00000001:3.0:1788575437.206453:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575437.206454:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:3.0:1788575437.206459:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206464:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575437.206466:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575437.206467:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575437.206469:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393209472, offset 0
00000400:00000200:3.0:1788575437.206471:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575437.206474:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575437.206477:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575437.206479:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575437.206481:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575437.206483:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575437.206484:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575437.206487:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330892:330892:256:4294967295] 192.168.202.110@tcp LPNI seq info [330849:330849:8:4294967295]
00000400:00000200:3.0:1788575437.206490:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575437.206494:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575437.206497:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575437.206500:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575437.206502:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575437.206505:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788575437.206508:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575437.206510:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1788575437.206512:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575437.206514:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575437.206516:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575437.206518:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575437.206520:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575437.206521:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575437.206523:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575437.206525:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206535:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575437.206537:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575462 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206541:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206542:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575437.206545:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575437.206547:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575437.206549:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575437.206551:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307040807
00000100:00000001:3.0:1788575437.206553:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307041721
00000100:00000001:3.0:1788575437.206554:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307040807
00000100:00000001:3.0:1788575437.206556:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307041721
00000100:00000001:3.0:1788575437.206558:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575437.206559:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206560:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575437.206562:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575437/real 1788575437]  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575437.206567:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575437.206568:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575437.206569:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575437.206570:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575437.206572:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575437.206573:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575437.206575:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575437.206580:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206584:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575437.206585:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206587:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575437.206588:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575437.206590:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206594:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575437.206596:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206601:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575437.206602:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206603:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575437.206604:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206605:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575437.206606:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575437.206608:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575437.206610:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575437.206611:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575437.206614:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575437.206619:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209472:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575437.206621:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575437.206623:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206627:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575437.206628:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575437.206630:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575437.206631:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575437.206632:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575437.206634:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209600:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575437.206636:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575437.206638:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575437.206639:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575437.206640:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575437.206642:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8000.
02000000:00000001:3.0:1788575437.206643:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575437.206645:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575437.206647:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575437.206648:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575437.206650:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393209600, portal 4
00000100:00000001:3.0:1788575437.206652:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575437.206653:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:3.0:1788575437.206657:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206661:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575437.206662:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575437.206664:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000200:3.0:1788575437.206665:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393209600, offset 0
00000400:00000200:3.0:1788575437.206667:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575437.206669:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575437.206671:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575437.206673:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575437.206675:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575437.206677:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575437.206678:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575437.206680:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330893:330893:256:4294967295] 192.168.202.110@tcp LPNI seq info [330850:330850:8:4294967295]
00000400:00000200:3.0:1788575437.206683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575437.206687:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575437.206689:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575437.206691:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575437.206694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575437.206696:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788575437.206699:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575437.206701:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1788575437.206703:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575437.206705:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:3.0:1788575437.206707:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575437.206709:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575437.206711:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:3.0:1788575437.206712:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:3.0:1788575437.206714:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575437.206716:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206722:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575437.206724:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575462 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206728:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206729:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575437.206731:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575437.206733:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575437.206735:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575437.206736:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307040953
00000100:00000001:3.0:1788575437.206738:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307041721
00000100:00000001:3.0:1788575437.206740:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307040953
00000100:00000001:3.0:1788575437.206741:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307041721
00000100:00000001:3.0:1788575437.206743:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575437.206744:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206745:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575437.206747:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575437/real 1788575437]  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575437.206752:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575437.206753:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575437.206754:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575437.206755:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575437.206756:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575437.206758:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575437.206760:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575437.206764:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206768:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575437.206769:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206770:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575437.206771:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575437.206773:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206777:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575437.206779:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206783:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575437.206785:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206786:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575437.206786:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206787:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575437.206789:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575437.206805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575437.206808:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575437.206810:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575437.206813:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575437.206828:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209600:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575437.206832:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206834:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575437.206837:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393209472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206845:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575437.206846:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575437.206849:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575437.206852:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575437.206853:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575437.206855:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575437.206857:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575437.206859:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575437.206862:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575437.206863:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575437.206865:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:3.0:1788575437.206868:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575437.206870:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206872:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575437.206875:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393209600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575462 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575437.206883:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575437.206884:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575437.206886:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8000.
02000000:00000001:3.0:1788575437.206888:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575437.206890:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575437.206891:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575437.206893:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575437.206895:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575437.206897:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575437.206898:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575437.206899:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:3.0:1788575437.206902:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575437.206904:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206906:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575437.206910:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575437.206911:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575437.206913:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575437.206915:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575437.589496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575437.589502:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575437.589504:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575437.589507:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575437.589511:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575437.589512:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575437.589515:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575437.589516:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575437.589519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575437.589520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575437.589522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575437.589523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575437.589577:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575437.589580:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575437.589583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575437.717477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575437.717483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575437.717485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575437.717489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575437.717492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575437.717495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575437.717497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575437.717499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575437.717501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575438.165477:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307041721, at 4307041728
00000100:00000040:1.0:1788575438.165477:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307041721, at 4307041728
00000100:00000010:0.0:1788575438.165491:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:1.0:1788575438.165494:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00010000:00000001:3.0:1788575438.613481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575438.613487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575438.613492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575438.613493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575438.613498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575438.613499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575438.613562:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575438.613565:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575438.613568:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575438.741471:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575438.741477:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575438.741479:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575438.741483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575438.741487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575438.741489:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575438.741492:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575438.741493:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575438.741496:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575439.637491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575439.637496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575439.637500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575439.637501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575439.637504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575439.637505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575439.637564:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575439.637569:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575439.637572:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575439.765490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575439.765498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575439.765500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575439.765506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575439.765509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575439.765513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575439.765517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575439.765519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575439.765523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575440.661501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575440.661509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575440.661515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575440.661516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575440.661520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575440.661522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575440.661591:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575440.661595:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575440.661600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575440.789524:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575440.789532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575440.789534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575440.789540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575440.789543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575440.789547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575440.789549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575440.789551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575440.789553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575441.685502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575441.685509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575441.685514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575441.685516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575441.685519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575441.685521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575441.685569:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575441.685572:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575441.685576:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575441.813512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575441.813520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575441.813523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575441.813529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575441.813533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575441.813538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575441.813541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575441.813544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575441.813548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575442.325487:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.0:1788575442.325498:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000100:00080000:2.0:1788575442.325500:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575442.325505:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:3.0:1788575442.325511:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000100:00080000:2.0:1788575442.325512:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575442.325519:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:3.0:1788575442.325520:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000100:00080000:2.0:1788575442.325524:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12373
00000400:00000200:2.0:1788575442.325533:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575442.325538:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575442.325540:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575442.325543:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325546:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575442.325549:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325552:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575442.325555:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575442.325557:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575442.325559:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575442.325562:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575442.325564:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575442.325566:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575442.325569:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325570:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575442.325572:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325574:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575442.325577:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575442.325579:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325581:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575442.325584:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575442.325586:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575442.325588:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575442.325590:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575442.325592:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575442.325597:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00e00.
00000020:00000040:2.0:1788575442.325600:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575442.325604:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575442.325605:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575442.325608:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575442.325610:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575442.325613:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575442.325620:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000001:2.0:1788575442.325625:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575442.325633:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575442.325641:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575442.325645:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575442.325649:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5f665f0] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575442.325666:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575442.325669:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575442.325673:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575442.325674:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575442.325676:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:1.0:1788575442.325679:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000001:2.0:1788575442.325679:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000001:0.0:1788575442.325682:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00080000:2.0:1788575442.325684:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12373
00000400:00000200:2.0:1788575442.325694:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000001:0.0:1788575442.325698:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1788575442.325699:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1788575442.325702:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1788575442.325705:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000001:0.0:1788575442.325706:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000001:3.0:1788575442.325708:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000800:00000001:0.0:1788575442.325709:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575442.325714:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575442.325714:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788575442.325720:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575442.325725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575442.325726:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325731:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575442.325737:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575442.325740:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575442.325756:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575442.325760:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575442.325762:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1788575442.325763:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575442.325767:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575442.325767:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575442.325774:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325776:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1788575442.325776:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1788575442.325780:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575442.325781:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575442.325782:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575442.325786:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575442.325788:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575442.325790:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325794:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575442.325797:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:3.0:1788575442.325799:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393209728:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1788575442.325802:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575442.325808:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575442.325808:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1788575442.325809:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575442.325815:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575442.325816:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:2.0:1788575442.325818:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575442.325821:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575442.325821:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575442.325828:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01c00.
02000000:00000001:3.0:1788575442.325828:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1788575442.325831:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575442.325834:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575442.325835:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575442.325837:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:3.0:1788575442.325837:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575442.325840:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575442.325844:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575442.325844:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575442.325853:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000400:00000200:3.0:1788575442.325857:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575442.325858:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575442.325865:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575442.325865:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000100:00000001:2.0:1788575442.325874:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575442.325877:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575442.325877:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000040:2.0:1788575442.325883:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000de3acaa1] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575442.325888:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393209728, portal 25
00000100:00000001:2.0:1788575442.325891:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575442.325901:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575442.325903:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1788575442.325904:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575442.325907:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1788575442.325907:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00080000:2.0:1788575442.325910:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575442.325915:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575442.325920:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12373
00000100:00000040:3.0:1788575442.325922:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575442.325924:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575442.325930:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575442.325934:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575442.325941:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325945:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575442.325949:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325954:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575442.325958:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575442.325960:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575442.325963:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575442.325964:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:2.0:1788575442.325965:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575442.325968:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575442.325972:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575442.325973:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575442.325977:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325979:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1788575442.325980:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526ee0.
00000100:00000040:2.0:1788575442.325981:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325987:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575442.325988:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000200:3.0:1788575442.325989:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393209728, offset 0
00000100:00080000:2.0:1788575442.325991:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575442.325997:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575442.325998:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575442.326000:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575442.326001:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575442.326004:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575442.326006:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00100000:2.0:1788575442.326007:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575442.326011:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575442.326012:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575442.326015:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00a80.
00000020:00000040:2.0:1788575442.326016:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575442.326020:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1788575442.326021:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575442.326022:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575442.326026:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575442.326027:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575442.326029:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575442.326031:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000100:00000001:2.0:1788575442.326035:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575442.326035:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:2.0:1788575442.326043:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575442.326046:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788575442.326052:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575442.326055:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1788575442.326056:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575442.326062:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000389a8694] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575442.326065:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1788575442.326073:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575442.326073:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:2.0:1788575442.326079:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575442.326084:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1788575442.326084:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330894:330894:256:4294967295] 192.168.202.110@tcp LPNI seq info [330851:330851:8:4294967295]
00000100:00000040:2.0:1788575442.326089:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12383)
00000400:00000200:3.0:1788575442.326094:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575442.326108:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575442.326115:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575442.326121:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728be00.
00000800:00000200:3.0:1788575442.326128:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575442.326132:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575442.326149:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575442.326153:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575442.326157:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307046841
00000100:00000010:3.0:1788575442.326163:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575442.326170:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575442.326176:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307046841
00000100:00000010:3.0:1788575442.326182:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575442.326186:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575442.326191:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575442.326195:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575442.326198:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575442.326201:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575442.326209:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575442.326212:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1788575442.326214:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575442.326216:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575442.326218:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575442.326223:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575442.326232:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575442.326234:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575442.326237:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575442.326239:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575442.326242:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575442.326245:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209856:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575442.326249:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575442.326251:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575442.326253:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575442.326255:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575442.326260:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575442.326262:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575442.326268:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575442.326272:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c52c0.
00000400:00000010:3.0:1788575442.326276:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575442.326279:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393209856, portal 10
00000100:00000001:3.0:1788575442.326282:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575442.326283:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:3.0:1788575442.326291:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575442.326299:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575442.326302:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575442.326305:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526f68.
00000100:00000200:3.0:1788575442.326307:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393209856, offset 0
00000400:00000200:3.0:1788575442.326312:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575442.326317:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575442.326321:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575442.326325:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575442.326328:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575442.326331:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575442.326334:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575442.326337:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330895:330895:255:4294967295] 192.168.202.110@tcp LPNI seq info [330852:330852:7:4294967295]
00000400:00000200:3.0:1788575442.326341:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575442.326348:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575442.326353:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575442.326357:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b800.
00000800:00000200:3.0:1788575442.326362:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575442.326366:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575442.326369:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575442.326372:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575442.326376:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575442.326379:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307045926
00000100:00000001:3.0:1788575442.326382:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307046841
00000100:00000001:3.0:1788575442.326384:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307045926
00000100:00000001:3.0:1788575442.326387:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307046841
00000100:00000001:3.0:1788575442.326412:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575442.326416:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575442.326418:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575442.326423:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575442.326432:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575442.326433:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575442.326436:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575442.326438:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575442.326440:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575442.326443:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575442.326446:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575442.326449:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575442.326450:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575442.326452:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575442.326457:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575442.326459:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575442.326462:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575442.326465:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575442.326469:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575442.326472:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393209984, portal 4
00000100:00000001:3.0:1788575442.326474:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575442.326475:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:3.0:1788575442.326482:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575442.326489:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575442.326492:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575442.326495:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526a18.
00000100:00000200:3.0:1788575442.326497:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393209984, offset 0
00000400:00000200:3.0:1788575442.326502:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575442.326506:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575442.326510:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575442.326513:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575442.326516:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575442.326519:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575442.326521:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575442.326524:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330896:330896:254:4294967295] 192.168.202.110@tcp LPNI seq info [330853:330853:6:4294967295]
00000400:00000200:3.0:1788575442.326529:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575442.326536:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575442.326540:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575442.326545:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b600.
00000800:00000200:3.0:1788575442.326549:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575442.326554:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575442.326557:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575442.326560:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575442.326565:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575442.326567:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307046073
00000100:00000001:3.0:1788575442.326570:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307046841
00000100:00000001:3.0:1788575442.326573:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307046073
00000100:00000001:3.0:1788575442.326575:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307046841
00000100:00000001:3.0:1788575442.326577:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575442.326580:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575442.326583:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575442.326585:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575442.326589:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575442.326591:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1788575442.326593:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575442.326595:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575442.326597:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575442.326599:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575442.709470:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575442.709478:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575442.709485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575442.709486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575442.709490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575442.709492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575442.709546:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575442.709549:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575442.709552:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575442.837510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575442.837518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575442.837521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575442.837528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575442.837533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575442.837538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575442.837541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575442.837544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575442.837548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575443.285520:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307046841, at 4307046848
00000100:00000040:1.0:1788575443.285520:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307046841, at 4307046848
00000100:00000010:1.0:1788575443.285537:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788575443.285540:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575443.733495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575443.733500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575443.733504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575443.733505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575443.733507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575443.733508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575443.733556:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575443.733559:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575443.733562:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575443.861469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575443.861475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575443.861478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575443.861483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575443.861486:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575443.861490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575443.861494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575443.861497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575443.861501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575444.757522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575444.757528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575444.757533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575444.757536:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575444.757539:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575444.757542:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575444.757545:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575444.757547:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575444.757549:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575444.757551:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575444.757554:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575444.757556:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575444.757557:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575444.757558:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575444.757560:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575444.757561:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575444.757563:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575444.757564:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575444.757617:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575444.757620:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575444.757623:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575444.885531:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575444.885536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575444.885538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575444.885541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575444.885544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575444.885547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575444.885549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575444.885552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575444.885554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788575445.397605:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788575445.397645:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788575445.397653:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575445.397658:0:2351:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788575445.397662:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575445.397666:0:2351:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788575445.397671:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575445.397675:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728be00.
00000400:00000200:1.0:1788575445.397680:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575445.397685:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575445.397702:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526ee0
00000400:00000010:1.0:1788575445.397708:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526ee0.
00000100:00000001:1.0:1788575445.397740:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575445.397750:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575445.397772:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575445.397777:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575467 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575445.397786:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575445.397788:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575445.397799:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575445.397801:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b800.
00000400:00000200:1.0:1788575445.397804:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575445.397808:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:3.0:1788575445.397809:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:1.0:1788575445.397811:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526f68
00000400:00000010:1.0:1788575445.397813:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526f68.
00000100:00000001:3.0:1788575445.397813:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575445.397815:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:3.0:1788575445.397816:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000200:1.0:1788575445.397819:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575445.397822:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575442/real 1788575445]  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575445.397827:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575445.397830:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575467 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575445.397835:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1788575445.397835:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1788575445.397837:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000400:00000001:3.0:1788575445.397837:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575445.397840:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000800:00000100:1.0:1788575445.397843:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000400:00000200:3.0:1788575445.397843:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000800:00000010:1.0:1788575445.397845:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b600.
00000400:00000010:3.0:1788575445.397845:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000400:00000200:1.0:1788575445.397848:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:3.0:1788575445.397848:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000400:00000200:1.0:1788575445.397851:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000100:00000200:3.0:1788575445.397852:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788575445.397853:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526a18
00000400:00000010:1.0:1788575445.397855:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526a18.
00000100:00000001:1.0:1788575445.397857:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575445.397861:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575445.397861:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575445.397869:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1788575445.397869:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575445.397870:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788575445.397871:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575467 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.397873:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575445.397875:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575445.397877:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575445.397878:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000100:00100000:3.0:1788575445.397879:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.397885:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575445.397890:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.397898:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575445.397901:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.397903:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575445.397904:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.397906:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575445.397909:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575445.397913:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575445.397918:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575445.397920:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575445.397925:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575445.397933:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393209728:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575445.397938:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575445.397940:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575445.397943:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393209728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.397949:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575445.397952:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575445.397956:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8000.
02000000:00000001:3.0:1788575445.397960:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575445.397963:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575445.397965:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575445.397967:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575445.397969:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575445.397973:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575445.397975:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575445.397977:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:3.0:1788575445.397981:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575445.397982:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.397985:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575445.397987:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575445.397988:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575445.397990:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575445.397994:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575442/real 1788575445]  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575445.398003:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575445.398004:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575445.398006:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c52c0.
00000400:00000200:3.0:1788575445.398008:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575445.398010:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575445.398016:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575445.398018:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575445.398026:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.398031:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575445.398033:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398034:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575445.398035:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575445.398039:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.398045:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575445.398049:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.398055:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575445.398058:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398059:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575445.398061:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398062:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575445.398064:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575445.398067:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575445.398071:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575445.398073:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575445.398077:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575445.398084:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209856:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575445.398087:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575445.398091:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575442/real 1788575445]  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575445.398097:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575445.398098:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575445.398100:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575445.398102:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575445.398104:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575445.398106:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575445.398108:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575445.398115:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.398121:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575445.398122:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398123:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575445.398124:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575445.398128:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.398133:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575445.398136:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.398146:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575445.398148:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398149:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575445.398150:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398152:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575445.398153:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575445.398155:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575445.398159:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575445.398161:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575445.398164:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575445.398170:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393209984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575445.398174:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398182:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575445.398185:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393209856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.398191:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575445.398192:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575445.398193:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575445.398196:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575445.398198:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575445.398199:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575445.398200:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575445.398202:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575445.398204:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575445.398205:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575445.398206:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:3.0:1788575445.398208:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575445.398209:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398212:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575445.398214:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393209984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575467 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575445.398219:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575445.398221:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575445.398222:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575445.398224:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575445.398225:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575445.398226:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575445.398227:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575445.398229:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575445.398230:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575445.398231:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575445.398233:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575445.398235:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575445.398236:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398237:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575445.398245:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575445.398247:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575445.398249:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575445.398250:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575445.781483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575445.781487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575445.781490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575445.781491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575445.781494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575445.781495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575445.781537:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575445.781541:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575445.781544:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575445.909502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575445.909507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575445.909508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575445.909511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575445.909514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575445.909517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575445.909518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575445.909520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575445.909522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575446.805490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575446.805498:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575446.805499:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575446.805501:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575446.805505:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575446.805507:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575446.805508:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575446.805509:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575446.805512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575446.805513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575446.805515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575446.805516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575446.805561:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575446.805564:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575446.805566:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575446.933467:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575446.933472:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575446.933474:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575446.933477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575446.933480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575446.933482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575446.933484:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575446.933486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575446.933488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575447.445483:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575447.445491:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575447.445493:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575447.445496:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575447.445500:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575447.445506:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12378
00000400:00000200:2.0:1788575447.445513:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575447.445518:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575447.445521:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575447.445523:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445525:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575447.445527:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445529:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575447.445531:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575447.445533:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575447.445535:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575447.445537:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575447.445539:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575447.445541:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575447.445543:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445544:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575447.445545:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445546:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575447.445548:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575447.445551:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445552:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575447.445554:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575447.445557:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575447.445559:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575447.445560:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575447.445562:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575447.445566:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01f80.
00000020:00000040:2.0:1788575447.445569:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575447.445571:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575447.445572:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575447.445574:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575447.445577:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575447.445580:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575447.445588:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788575447.445593:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575447.445601:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575447.445611:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575447.445615:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575447.445619:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8c8ec7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575447.445647:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575447.445650:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575447.445653:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575447.445655:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575447.445656:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575447.445658:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575447.445661:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12378
00000400:00000200:2.0:1788575447.445665:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575447.445668:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575447.445670:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575447.445672:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445674:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575447.445676:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445678:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575447.445680:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575447.445681:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575447.445683:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575447.445685:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575447.445687:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575447.445689:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575447.445692:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445704:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575447.445707:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445710:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575447.445712:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575447.445715:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445717:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575447.445719:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575447.445724:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575447.445726:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575447.445728:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575447.445732:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575447.445734:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575447.445736:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575447.445739:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03800.
00000020:00000040:2.0:1788575447.445742:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575447.445744:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575447.445745:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575447.445746:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575447.445747:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575447.445749:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575447.445753:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:2.0:1788575447.445757:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575447.445762:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575447.445769:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575447.445771:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575447.445775:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014d98894] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575447.445783:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575447.445786:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575447.445790:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575447.445791:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575447.445793:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575447.445795:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575447.445798:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12378
00000400:00000200:2.0:1788575447.445802:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575447.445805:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575447.445810:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575447.445812:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445814:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575447.445816:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445818:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575447.445820:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575447.445821:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575447.445823:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575447.445825:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575447.445827:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575447.445828:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575447.445831:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445833:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575447.445835:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445837:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575447.445839:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575447.445842:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575447.445844:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575447.445845:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575447.445848:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575447.445850:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575447.445852:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575447.445854:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575447.445856:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575447.445857:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575447.445861:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02a00.
00000020:00000040:2.0:1788575447.445863:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575447.445866:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575447.445867:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575447.445868:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575447.445869:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575447.445871:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575447.445874:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788575447.445877:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575447.445882:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575447.445889:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575447.445891:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575447.445894:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cce910e4] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575447.445900:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575447.445903:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575447.445906:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575447.445911:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12388)
00000100:00000001:3.0:1788575447.445957:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575447.445962:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575447.445964:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575447.445971:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.445983:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575447.445985:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575447.445989:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575447.445992:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575447.445994:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575447.445998:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393210112:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575447.446002:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575447.446006:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575447.446009:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575447.446011:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575447.446018:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575447.446020:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575447.446025:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575447.446029:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575447.446035:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575447.446042:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393210112, portal 25
00000100:00000001:3.0:1788575447.446045:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575447.446046:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:3.0:1788575447.446053:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446060:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575447.446063:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575447.446066:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575447.446069:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393210112, offset 0
00000400:00000200:3.0:1788575447.446089:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575447.446097:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575447.446104:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575447.446110:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575447.446114:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575447.446117:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575447.446120:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575447.446124:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330897:330897:256:4294967295] 192.168.202.110@tcp LPNI seq info [330854:330854:8:4294967295]
00000400:00000200:3.0:1788575447.446129:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575447.446138:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575447.446144:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575447.446150:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575447.446157:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575447.446160:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575447.446166:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575447.446170:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575447.446174:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575447.446176:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575447.446181:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575447.446186:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575447.446193:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575447.446195:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575447.446200:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575447.446204:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446215:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575447.446296:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575472 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446304:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575447.446306:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575447.446314:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575447.446319:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575447.446322:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307051961
00000100:00000010:3.0:1788575447.446327:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575447.446333:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575447.446340:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307051961
00000100:00000010:3.0:1788575447.446343:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575447.446346:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575447.446349:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575447.446351:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575447.446354:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575447.446358:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575447/real 1788575447]  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575447.446368:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575447.446370:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575447.446372:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575447.446375:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575447.446377:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575447.446380:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575447.446383:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575447.446422:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446429:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575447.446431:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.446433:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575447.446436:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575447.446440:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446445:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575447.446449:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446455:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575447.446458:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.446459:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575447.446461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.446463:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575447.446466:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575447.446468:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575447.446472:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575447.446475:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575447.446479:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575447.446486:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393210112:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575447.446490:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575447.446493:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446500:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575447.446501:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575447.446503:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575447.446505:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575447.446507:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575447.446509:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393210240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575447.446513:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575447.446515:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575447.446516:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575447.446518:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575447.446522:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575447.446524:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575447.446528:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575447.446530:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575447.446534:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575447.446537:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393210240, portal 10
00000100:00000001:3.0:1788575447.446539:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575447.446541:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:3.0:1788575447.446548:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446556:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575447.446558:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575447.446560:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575447.446562:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393210240, offset 0
00000400:00000200:3.0:1788575447.446565:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575447.446569:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575447.446573:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575447.446576:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575447.446578:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575447.446580:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575447.446582:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575447.446585:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330898:330898:256:4294967295] 192.168.202.110@tcp LPNI seq info [330855:330855:8:4294967295]
00000400:00000200:3.0:1788575447.446589:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575447.446595:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575447.446599:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575447.446603:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575447.446606:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575447.446609:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575447.446612:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575447.446615:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575447.446618:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575447.446619:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575447.446622:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575447.446626:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575447.446628:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575447.446630:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575447.446632:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575447.446636:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446643:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575447.446646:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575472 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446652:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575447.446653:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575447.446656:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575447.446660:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575447.446662:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575447.446664:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307051047
00000100:00000001:3.0:1788575447.446667:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307051961
00000100:00000001:3.0:1788575447.446669:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307051047
00000100:00000001:3.0:1788575447.446671:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307051961
00000100:00000001:3.0:1788575447.446673:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575447.446675:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575447.446676:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575447.446680:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575447/real 1788575447]  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575447.446686:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575447.446688:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575447.446689:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575447.446691:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575447.446693:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575447.446706:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575447.446710:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575447.446718:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446723:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575447.446724:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.446726:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575447.446727:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575447.446731:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446737:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575447.446741:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446748:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575447.446750:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.446751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575447.446752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.446754:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575447.446756:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575447.446759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575447.446762:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575447.446764:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575447.446767:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575447.446774:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393210240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575447.446778:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575447.446781:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446787:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575447.446788:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575447.446790:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575447.446792:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575447.446794:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575447.446796:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393210368:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575447.446799:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575447.446801:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575447.446803:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575447.446804:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575447.446808:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8000.
02000000:00000001:3.0:1788575447.446810:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575447.446813:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575447.446816:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575447.446820:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575447.446822:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393210368, portal 4
00000100:00000001:3.0:1788575447.446824:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575447.446825:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:3.0:1788575447.446831:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446838:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575447.446840:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575447.446842:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575447.446845:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393210368, offset 0
00000400:00000200:3.0:1788575447.446848:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575447.446852:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575447.446855:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575447.446857:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575447.446860:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575447.446862:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575447.446864:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575447.446867:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330899:330899:256:4294967295] 192.168.202.110@tcp LPNI seq info [330856:330856:8:4294967295]
00000400:00000200:3.0:1788575447.446870:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575447.446876:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575447.446879:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575447.446883:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575447.446886:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575447.446888:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575447.446892:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575447.446895:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575447.446898:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575447.446900:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575447.446903:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575447.446905:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575447.446908:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575447.446910:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575447.446911:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575447.446914:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446921:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575447.446924:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575472 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446929:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575447.446930:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575447.446933:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575447.446936:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575447.446938:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575447.446940:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307051193
00000100:00000001:3.0:1788575447.446942:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307051961
00000100:00000001:3.0:1788575447.446944:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307051193
00000100:00000001:3.0:1788575447.446946:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307051961
00000100:00000001:3.0:1788575447.446948:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575447.446950:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575447.446951:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575447.446955:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575447/real 1788575447]  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575447.446962:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575447.446963:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575447.446964:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575447.446967:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575447.446968:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575447.446971:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575447.446974:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575447.446982:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.446988:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575447.446989:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.446991:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575447.446992:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575447.446995:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.447001:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575447.447005:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.447012:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575447.447014:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.447016:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575447.447017:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.447019:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575447.447021:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575447.447024:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575447.447028:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575447.447030:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575447.447033:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575447.447039:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393210368:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575447.447043:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.447045:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575447.447049:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393210112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.447054:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575447.447057:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575447.447060:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9800.
02000000:00000001:3.0:1788575447.447063:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575447.447065:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575447.447066:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575447.447068:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575447.447071:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575447.447074:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575447.447076:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575447.447078:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575447.447081:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575447.447082:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.447083:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575447.447086:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393210240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.447092:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575447.447093:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575447.447094:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf9400.
02000000:00000001:3.0:1788575447.447096:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575447.447097:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575447.447098:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575447.447100:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575447.447101:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575447.447103:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575447.447104:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575447.447105:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:3.0:1788575447.447107:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575447.447108:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.447109:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575447.447112:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393210368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575472 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575447.447118:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575447.447120:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575447.447121:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8000.
02000000:00000001:3.0:1788575447.447130:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575447.447131:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575447.447132:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575447.447134:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575447.447135:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575447.447137:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575447.447138:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575447.447139:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:3.0:1788575447.447141:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575447.447142:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.447144:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575447.447151:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575447.447153:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575447.447156:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575447.447158:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575447.829472:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575447.829477:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575447.829482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575447.829483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575447.829487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575447.829488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575447.829530:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575447.829533:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575447.829537:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575447.957494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575447.957499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575447.957501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575447.957504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575447.957507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575447.957510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575447.957512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575447.957513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575447.957515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575448.405493:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307051961, at 4307051968
00000100:00000040:0.0:1788575448.405501:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307051961, at 4307051968
00000100:00000010:1.0:1788575448.405507:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575448.405511:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575448.853494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575448.853499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575448.853505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575448.853506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575448.853509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575448.853510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575448.853563:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575448.853566:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575448.853568:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575448.981466:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575448.981472:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575448.981474:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575448.981477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575448.981480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575448.981482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575448.981484:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575448.981486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575448.981488:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575449.877494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575449.877500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575449.877506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575449.877507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575449.877510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575449.877512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575449.877541:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575449.877545:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575449.877548:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575450.005489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575450.005496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575450.005498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575450.005502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575450.005506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575450.005510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575450.005512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575450.005514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575450.005519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575450.901530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575450.901537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575450.901543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575450.901545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575450.901549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575450.901550:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575450.901616:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575450.901619:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575450.901622:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575451.029452:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575451.029459:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575451.029462:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575451.029467:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575451.029471:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575451.029474:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575451.029477:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575451.029480:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575451.029484:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575451.925503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575451.925508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575451.925513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575451.925515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575451.925518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575451.925519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575451.925581:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575451.925585:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575451.925588:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575452.053481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575452.053487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575452.053489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575452.053494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575452.053497:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575452.053502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575452.053504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575452.053506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575452.053509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575452.565500:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575452.565511:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575452.565514:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575452.565518:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575452.565523:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575452.565530:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12383
00000400:00000200:3.0:1788575452.565538:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575452.565544:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575452.565547:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575452.565551:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565554:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575452.565557:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565559:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575452.565563:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575452.565565:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575452.565568:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575452.565570:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575452.565572:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575452.565575:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575452.565577:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565579:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575452.565580:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565582:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575452.565585:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575452.565589:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565591:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575452.565594:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575452.565598:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575452.565600:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575452.565602:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575452.565604:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575452.565613:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788575452.565616:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575452.565619:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575452.565621:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575452.565623:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575452.565626:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575452.565630:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575452.565636:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575452.565642:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575452.565679:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575452.565689:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575452.565693:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575452.565697:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575452.565714:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575452.565717:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575452.565719:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575452.565720:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575452.565721:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575452.565723:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575452.565725:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12383
00000400:00000200:3.0:1788575452.565728:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575452.565731:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575452.565732:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575452.565734:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565736:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575452.565738:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565739:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575452.565741:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575452.565742:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1788575452.565742:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575452.565746:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575452.565748:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1788575452.565748:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575452.565752:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575452.565752:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575452.565754:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575452.565757:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565758:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575452.565762:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575452.565763:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575452.565764:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575452.565767:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575452.565771:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565775:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575452.565777:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575452.565782:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575452.565787:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575452.565790:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575452.565794:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:3.0:1788575452.565794:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788575452.565796:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788575452.565798:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575452.565802:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575452.565804:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:3.0:1788575452.565805:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
02000000:00000001:2.0:1788575452.565808:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1788575452.565808:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575452.565812:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575452.565814:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575452.565814:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575452.565819:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575452.565822:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:2.0:1788575452.565823:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393210496:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575452.565825:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575452.565830:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:3.0:1788575452.565830:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575452.565833:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575452.565836:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:3.0:1788575452.565839:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575452.565840:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575452.565846:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575452.565850:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575452.565851:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575452.565853:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
00000100:00000040:3.0:1788575452.565854:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575452.565857:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575452.565866:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575452.565867:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575452.565868:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575452.565872:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:2.0:1788575452.565873:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000001:3.0:1788575452.565873:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575452.565875:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575452.565879:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575452.565881:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12383
00000400:00000010:2.0:1788575452.565882:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000400:00000200:3.0:1788575452.565883:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575452.565888:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575452.565889:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575452.565891:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575452.565892:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393210496, portal 25
00000100:00000001:3.0:1788575452.565893:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575452.565897:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1788575452.565897:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565898:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575452.565900:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000020:00000001:3.0:1788575452.565901:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575452.565903:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575452.565905:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575452.565907:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575452.565908:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575452.565910:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575452.565911:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575452.565914:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565917:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575452.565920:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565923:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575452.565927:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788575452.565930:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1788575452.565931:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575452.565934:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575452.565936:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575452.565937:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575452.565938:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575452.565941:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575452.565942:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000001:3.0:1788575452.565943:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575452.565945:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000200:2.0:1788575452.565948:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393210496, offset 0
00000008:00000001:3.0:1788575452.565948:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575452.565949:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575452.565955:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575452.565957:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575452.565960:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575452.565961:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575452.565963:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575452.565965:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575452.565966:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575452.565967:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575452.565970:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000400:00000200:2.0:1788575452.565974:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575452.565974:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575452.565981:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575452.565985:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788575452.565991:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575452.565993:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575452.565995:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1788575452.565997:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575452.566002:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575452.566005:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575452.566008:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1788575452.566008:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575452.566013:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575452.566015:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:3.0:1788575452.566019:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12393)
00000400:00000200:2.0:1788575452.566021:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330900:330900:256:4294967295] 192.168.202.110@tcp LPNI seq info [330857:330857:8:4294967295]
00000400:00000200:2.0:1788575452.566028:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575452.566038:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575452.566043:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575452.566047:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575452.566054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575452.566057:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575452.566063:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575452.566066:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575452.566070:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575452.566072:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575452.566078:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575452.566083:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575452.566088:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575452.566090:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575452.566094:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575452.566098:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566105:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575452.566109:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575452.566117:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575452.566122:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575452.566125:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575452.566128:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307057081
00000100:00000010:2.0:1788575452.566133:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cea0.
00000100:00000001:2.0:1788575452.566139:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575452.566144:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307057081
00000100:00000010:2.0:1788575452.566147:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c180.
00000100:00000001:2.0:1788575452.566150:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575452.566153:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575452.566155:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575452.566158:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575452.566162:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575452/real 1788575452]  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575452.566170:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575452.566171:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575452.566173:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575452.566176:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575452.566177:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575452.566179:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575452.566183:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575452.566190:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566196:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575452.566198:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566200:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575452.566202:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575452.566205:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566212:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575452.566216:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566222:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575452.566224:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566225:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575452.566226:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566229:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575452.566230:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575452.566233:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575452.566237:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575452.566239:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575452.566243:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575452.566250:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393210496:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575452.566254:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566256:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575452.566259:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393210496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566264:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575452.566266:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575452.566269:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575452.566272:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575452.566273:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575452.566274:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575452.566276:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575452.566279:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575452.566282:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575452.566283:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575452.566285:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575452.566287:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575452.566288:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566290:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566297:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575452.566299:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575452.566301:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575452.566302:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575452.566304:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575452.566307:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566314:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575452.566315:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575452.566317:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575452.566318:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575452.566320:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575452.566323:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393210624:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575452.566326:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575452.566327:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575452.566328:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575452.566330:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575452.566332:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575452.566334:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575452.566338:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575452.566340:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575452.566343:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575452.566346:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393210624, portal 10
00000100:00000001:2.0:1788575452.566348:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575452.566349:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:2.0:1788575452.566355:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566361:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575452.566363:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575452.566365:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575452.566368:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393210624, offset 0
00000400:00000200:2.0:1788575452.566371:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575452.566375:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575452.566378:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575452.566380:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575452.566383:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575452.566385:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575452.566387:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575452.566407:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330901:330901:256:4294967295] 192.168.202.110@tcp LPNI seq info [330858:330858:8:4294967295]
00000400:00000200:2.0:1788575452.566412:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575452.566418:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575452.566421:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575452.566425:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575452.566429:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575452.566431:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575452.566435:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575452.566437:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575452.566442:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575452.566443:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575452.566446:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575452.566449:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575452.566452:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575452.566454:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575452.566456:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575452.566459:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566466:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575452.566469:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566475:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575452.566476:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575452.566479:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575452.566482:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575452.566485:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575452.566487:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307056167
00000100:00000001:2.0:1788575452.566489:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307057081
00000100:00000001:2.0:1788575452.566491:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307056167
00000100:00000001:2.0:1788575452.566493:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307057081
00000100:00000001:2.0:1788575452.566495:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575452.566496:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575452.566498:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575452.566501:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575452/real 1788575452]  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575452.566514:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575452.566515:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575452.566517:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575452.566520:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575452.566521:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575452.566523:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575452.566525:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575452.566533:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566538:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575452.566539:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566540:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575452.566542:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575452.566545:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566551:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575452.566554:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566560:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575452.566562:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566563:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575452.566564:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566566:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575452.566568:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575452.566570:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575452.566573:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575452.566575:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575452.566579:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575452.566586:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393210624:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575452.566589:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575452.566592:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566598:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575452.566599:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575452.566601:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575452.566603:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575452.566604:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575452.566607:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393210752:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575452.566609:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575452.566611:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575452.566612:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575452.566614:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575452.566617:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575452.566619:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575452.566622:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575452.566624:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575452.566627:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575452.566629:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393210752, portal 4
00000100:00000001:2.0:1788575452.566631:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575452.566632:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788575452.566637:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566643:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575452.566644:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575452.566659:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575452.566662:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393210752, offset 0
00000400:00000200:2.0:1788575452.566666:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575452.566670:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575452.566673:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575452.566675:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575452.566677:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575452.566679:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575452.566682:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575452.566684:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330902:330902:256:4294967295] 192.168.202.110@tcp LPNI seq info [330859:330859:8:4294967295]
00000400:00000200:2.0:1788575452.566688:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575452.566694:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575452.566697:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575452.566701:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575452.566704:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575452.566706:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575452.566711:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575452.566713:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575452.566716:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575452.566718:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575452.566720:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575452.566723:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575452.566726:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575452.566728:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575452.566730:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575452.566733:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566740:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575452.566743:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566748:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575452.566750:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575452.566752:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575452.566754:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575452.566757:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575452.566760:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307056313
00000100:00000001:2.0:1788575452.566762:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307057081
00000100:00000001:2.0:1788575452.566764:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307056313
00000100:00000001:2.0:1788575452.566765:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307057081
00000100:00000001:2.0:1788575452.566767:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575452.566769:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575452.566770:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575452.566774:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575452/real 1788575452]  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575452.566780:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575452.566781:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575452.566783:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575452.566785:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575452.566786:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575452.566787:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575452.566790:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575452.566797:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566802:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575452.566803:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566805:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575452.566806:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575452.566809:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566815:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575452.566819:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566824:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575452.566826:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566828:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575452.566829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566831:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575452.566832:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575452.566835:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575452.566838:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575452.566839:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575452.566843:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575452.566849:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393210752:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575452.566852:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566853:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575452.566856:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393210624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566861:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575452.566862:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575452.566864:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575452.566866:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575452.566867:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575452.566869:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575452.566871:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575452.566872:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575452.566874:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575452.566875:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575452.566877:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575452.566879:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575452.566880:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566881:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575452.566884:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393210752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575452.566894:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575452.566895:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575452.566896:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575452.566898:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575452.566899:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575452.566900:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575452.566902:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575452.566903:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575452.566904:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575452.566905:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575452.566906:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575452.566908:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575452.566909:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566910:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575452.566913:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575452.566914:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575452.566915:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575452.566917:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575452.949515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575452.949520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575452.949523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575452.949524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575452.949527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575452.949527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575452.949574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575452.949577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575452.949579:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575453.077451:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575453.077457:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575453.077458:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575453.077463:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575453.077465:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575453.077468:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575453.077470:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575453.077472:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575453.077474:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575453.525482:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307057081, at 4307057088
00000100:00000040:1.0:1788575453.525482:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307057081, at 4307057088
00000100:00000010:1.0:1788575453.525496:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000100:00000010:0.0:1788575453.525500:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00010000:00000001:3.0:1788575453.973485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575453.973492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575453.973497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575453.973498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575453.973501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575453.973502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575453.973550:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575453.973553:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575453.973556:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575454.101487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575454.101493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575454.101494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575454.101498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575454.101501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575454.101504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575454.101505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575454.101508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575454.101509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575454.997505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575454.997511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575454.997514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575454.997517:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575454.997518:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575454.997521:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575454.997523:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575454.997524:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575454.997525:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575454.997527:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575454.997529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575454.997530:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575454.997530:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575454.997531:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575454.997532:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575454.997533:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575454.997534:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575454.997535:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575454.997582:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575454.997584:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575454.997587:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575455.125491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575455.125496:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575455.125498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575455.125502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575455.125505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575455.125507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575455.125509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575455.125511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575455.125513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575456.021454:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575456.021459:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575456.021462:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575456.021463:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575456.021466:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575456.021467:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575456.021521:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575456.021525:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575456.021528:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575456.149482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575456.149489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575456.149490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575456.149493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575456.149495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575456.149498:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575456.149500:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575456.149501:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575456.149503:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575457.045480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575457.045491:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575457.045493:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575457.045497:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575457.045503:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575457.045506:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575457.045509:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575457.045512:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575457.045518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575457.045519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575457.045523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575457.045525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575457.045573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575457.045578:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575457.045582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575457.173462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575457.173470:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575457.173471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575457.173476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575457.173479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575457.173483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575457.173485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575457.173487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575457.173489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575457.685504:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575457.685514:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575457.685516:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575457.685520:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575457.685524:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575457.685531:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12388
00000400:00000200:3.0:1788575457.685541:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575457.685546:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575457.685549:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575457.685551:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575457.685553:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575457.685555:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575457.685558:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575457.685562:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575457.685564:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575457.685567:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575457.685570:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575457.685573:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575457.685576:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575457.685580:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575457.685582:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575457.685584:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575457.685586:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575457.685589:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575457.685592:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575457.685595:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575457.685610:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575457.685614:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575457.685618:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575457.685621:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575457.685623:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575457.685634:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575457.685638:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575457.685643:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575457.685645:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575457.685648:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575457.685651:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575457.685656:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575457.685664:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575457.685672:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575457.685691:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575457.685703:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575457.685707:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575457.685712:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575457.685734:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575457.685737:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575457.685741:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575457.685742:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575457.685745:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575457.685749:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575457.685753:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12388
00000100:00000001:2.0:1788575457.685766:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:3.0:1788575457.685767:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575457.685773:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1788575457.685776:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1788575457.685778:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575457.685779:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575457.685787:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575457.685794:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575457.685795:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575457.685800:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575457.685810:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575457.685818:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575457.685819:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575457.685824:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575457.685824:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
02000000:00000001:2.0:1788575457.685831:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:3.0:1788575457.685831:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1788575457.685836:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1788575457.685837:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
02000000:00000001:2.0:1788575457.685840:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1788575457.685845:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00100000:2.0:1788575457.685848:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393210880:192.168.202.110@tcp:250:kworker.0
00000020:00000001:3.0:1788575457.685854:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575457.685856:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575457.685861:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575457.685866:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575457.685866:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575457.685875:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575457.685876:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575457.685882:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000010:2.0:1788575457.685884:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
00000100:00000001:3.0:1788575457.685890:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575457.685891:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575457.685901:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000400:00000200:2.0:1788575457.685909:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575457.685914:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00080000:3.0:1788575457.685914:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575457.685925:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575457.685928:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:2.0:1788575457.685934:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
02000000:00000001:3.0:1788575457.685936:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575457.685940:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575457.685943:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393210880, portal 25
00000100:00000001:3.0:1788575457.685943:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575457.685947:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575457.685951:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000008:00100000:3.0:1788575457.685951:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575457.685961:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575457.685962:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575457.685966:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.685978:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:3.0:1788575457.685978:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000100:00000040:2.0:1788575457.685986:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575457.685987:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:2.0:1788575457.685989:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575457.685996:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393210880, offset 0
02000000:00000001:3.0:1788575457.685997:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575457.686002:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575457.686008:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575457.686010:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575457.686014:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575457.686017:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575457.686017:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575457.686030:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000400:00000200:2.0:1788575457.686031:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788575457.686041:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575457.686042:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575457.686048:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575457.686055:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00100000:3.0:1788575457.686056:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575457.686060:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575457.686067:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330903:330903:256:4294967295] 192.168.202.110@tcp LPNI seq info [330860:330860:8:4294967295]
00000400:00000200:2.0:1788575457.686085:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575457.686095:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575457.686099:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575457.686107:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575457.686108:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575457.686113:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000100:00000040:3.0:1788575457.686119:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788575457.686126:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575457.686135:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575457.686149:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575457.686155:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:3.0:1788575457.686158:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1788575457.686160:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575457.686168:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000100:00080000:3.0:1788575457.686168:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575457.686173:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:3.0:1788575457.686175:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:2.0:1788575457.686177:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575457.686182:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575457.686187:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:3.0:1788575457.686187:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575457.686192:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575457.686194:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575457.686200:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575457.686200:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575457.686211:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12388
00000100:00000001:2.0:1788575457.686218:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000400:00000200:3.0:1788575457.686223:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575457.686224:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575457.686238:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1788575457.686239:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.686246:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00080000:3.0:1788575457.686246:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575457.686254:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575457.686255:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575457.686262:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575457.686264:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575457.686265:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307062201
00000100:00000010:2.0:1788575457.686272:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c060.
00000100:00000040:3.0:1788575457.686272:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575457.686281:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575457.686285:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:3.0:1788575457.686291:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575457.686294:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307062201
00000020:00000040:3.0:1788575457.686295:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000010:2.0:1788575457.686298:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c5a0.
00000020:00000001:3.0:1788575457.686300:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575457.686302:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000040:3.0:1788575457.686304:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:2.0:1788575457.686306:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575457.686311:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1788575457.686311:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788575457.686317:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000020:00000001:3.0:1788575457.686319:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00100000:2.0:1788575457.686324:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575457/real 1788575457]  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575457.686329:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575457.686336:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575457.686348:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575457.686354:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575457.686360:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575457.686369:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000001:2.0:1788575457.686371:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575457.686374:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1788575457.686375:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575457.686377:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
02000000:00000001:3.0:1788575457.686380:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575457.686381:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575457.686385:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
02000000:00000001:3.0:1788575457.686389:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575457.686430:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575457.686432:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575457.686435:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575457.686438:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.686440:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000001:3.0:1788575457.686440:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575457.686446:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000100:00000200:2.0:1788575457.686447:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575457.686452:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575457.686458:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575457.686462:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575457.686469:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:2.0:1788575457.686474:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575457.686474:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575457.686481:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575457.686489:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:2.0:1788575457.686490:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575457.686493:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575457.686494:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.686500:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575457.686502:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575457.686505:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:2.0:1788575457.686512:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575457.686533:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575457.686538:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575457.686550:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686556:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575457.686560:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575457.686562:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575457.686566:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575457.686574:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575457.686587:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:3.0:1788575457.686587:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12398)
00000100:00000001:2.0:1788575457.686598:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.686609:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575457.686611:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.686613:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575457.686615:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575457.686619:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575457.686625:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575457.686628:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575457.686633:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575457.686642:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393210880:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575457.686647:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.686649:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575457.686652:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393210880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686658:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575457.686660:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575457.686663:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575457.686666:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575457.686669:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575457.686671:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575457.686673:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575457.686675:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575457.686679:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575457.686681:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575457.686683:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575457.686686:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575457.686687:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.686689:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.686694:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575457.686696:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.686697:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575457.686698:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575457.686699:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575457.686702:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686708:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575457.686709:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575457.686710:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575457.686712:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575457.686713:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575457.686715:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575457.686717:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575457.686719:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575457.686720:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575457.686721:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575457.686723:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575457.686724:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575457.686727:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575457.686729:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575457.686732:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575457.686735:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393211008, portal 10
00000100:00000001:2.0:1788575457.686737:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575457.686738:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:2.0:1788575457.686744:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686751:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575457.686753:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575457.686755:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575457.686758:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393211008, offset 0
00000400:00000200:2.0:1788575457.686762:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575457.686766:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575457.686770:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575457.686772:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575457.686775:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575457.686777:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575457.686778:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575457.686780:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330904:330904:256:4294967295] 192.168.202.110@tcp LPNI seq info [330861:330861:8:4294967295]
00000400:00000200:2.0:1788575457.686783:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575457.686788:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575457.686791:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575457.686793:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575457.686795:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575457.686797:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575457.686800:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575457.686802:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1788575457.686804:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575457.686805:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575457.686807:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575457.686809:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575457.686811:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575457.686812:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575457.686813:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575457.686816:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686821:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575457.686822:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686826:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.686827:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575457.686829:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575457.686832:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575457.686834:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575457.686836:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307061287
00000100:00000001:2.0:1788575457.686837:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307062201
00000100:00000001:2.0:1788575457.686839:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307061287
00000100:00000001:2.0:1788575457.686840:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307062201
00000100:00000001:2.0:1788575457.686841:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575457.686842:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.686843:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575457.686845:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575457/real 1788575457]  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575457.686850:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575457.686851:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575457.686852:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575457.686853:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575457.686854:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575457.686856:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575457.686857:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575457.686862:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686865:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575457.686866:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.686867:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575457.686868:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575457.686869:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686873:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575457.686875:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686879:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575457.686880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.686881:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575457.686882:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.686883:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575457.686885:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575457.686886:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575457.686888:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575457.686889:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575457.686891:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575457.686895:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575457.686898:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575457.686900:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686904:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575457.686905:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575457.686907:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575457.686908:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575457.686909:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575457.686910:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575457.686912:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575457.686913:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575457.686915:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575457.686916:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575457.686918:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575457.686919:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575457.686921:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575457.686923:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575457.686924:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575457.686926:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393211136, portal 4
00000100:00000001:2.0:1788575457.686927:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575457.686928:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788575457.686932:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.686943:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575457.686945:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575457.686946:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575457.686948:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393211136, offset 0
00000400:00000200:2.0:1788575457.686953:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575457.686955:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575457.686957:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575457.686959:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575457.686961:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575457.686963:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575457.686964:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575457.686966:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330905:330905:256:4294967295] 192.168.202.110@tcp LPNI seq info [330862:330862:8:4294967295]
00000400:00000200:2.0:1788575457.686968:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575457.686972:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575457.686974:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575457.686976:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575457.686978:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575457.686980:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575457.686982:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575457.686984:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1788575457.686986:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575457.686987:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575457.686989:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575457.686991:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575457.686992:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575457.686993:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575457.686995:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575457.686997:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.687001:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575457.687003:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.687006:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.687007:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575457.687009:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575457.687011:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575457.687013:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575457.687014:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307061433
00000100:00000001:2.0:1788575457.687015:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307062201
00000100:00000001:2.0:1788575457.687017:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307061433
00000100:00000001:2.0:1788575457.687018:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307062201
00000100:00000001:2.0:1788575457.687019:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575457.687021:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.687022:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575457.687024:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575457/real 1788575457]  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575457.687028:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575457.687028:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575457.687029:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575457.687030:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575457.687031:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575457.687032:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575457.687034:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575457.687039:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.687043:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575457.687043:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.687044:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575457.687045:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575457.687047:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.687050:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575457.687052:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.687056:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575457.687057:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.687058:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575457.687058:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.687060:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575457.687061:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575457.687062:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575457.687064:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575457.687066:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575457.687068:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575457.687072:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575457.687074:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.687075:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575457.687076:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393211008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.687080:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575457.687081:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575457.687082:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575457.687083:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575457.687084:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575457.687085:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575457.687087:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575457.687088:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575457.687089:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575457.687090:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575457.687091:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:2.0:1788575457.687092:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575457.687093:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.687094:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575457.687096:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393211136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575457.687099:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575457.687100:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575457.687100:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575457.687102:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575457.687103:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575457.687103:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575457.687105:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575457.687105:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575457.687107:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575457.687107:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575457.687108:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575457.687109:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575457.687110:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.687111:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575457.687113:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575457.687114:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575457.687115:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575457.687116:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575458.069451:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575458.069457:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575458.069461:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575458.069462:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575458.069465:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575458.069466:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575458.069525:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575458.069529:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575458.069532:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575458.197521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575458.197531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575458.197534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575458.197542:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575458.197549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575458.197556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575458.197559:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575458.197564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575458.197568:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575458.645516:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307062201, at 4307062208
00000100:00000040:1.0:1788575458.645516:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307062201, at 4307062208
00000100:00000010:0.0:1788575458.645533:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000100:00000010:1.0:1788575458.645536:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00010000:00000001:3.0:1788575459.093490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575459.093496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575459.093501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575459.093502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575459.093505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575459.093506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575459.093565:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575459.093569:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575459.093573:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575459.221491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575459.221497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575459.221498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575459.221502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575459.221505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575459.221508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575459.221509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575459.221511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575459.221513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575460.117502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575460.117511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575460.117517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575460.117518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575460.117522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575460.117523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575460.117581:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575460.117584:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575460.117587:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575460.245554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575460.245564:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575460.245566:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575460.245573:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575460.245577:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575460.245581:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575460.245584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575460.245587:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575460.245590:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575461.141537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575461.141546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575461.141553:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575461.141554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575461.141558:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575461.141559:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575461.141652:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575461.141658:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575461.141674:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575461.269508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575461.269518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575461.269520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575461.269527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575461.269532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575461.269537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575461.269540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575461.269542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575461.269546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575462.165490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575462.165497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575462.165502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575462.165503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575462.165506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575462.165507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575462.165568:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575462.165571:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575462.165574:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575462.293523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575462.293535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575462.293538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575462.293544:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575462.293548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575462.293553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575462.293556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575462.293559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575462.293564:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575462.805519:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575462.805529:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575462.805532:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575462.805536:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575462.805541:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575462.805548:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12393
00000400:00000200:3.0:1788575462.805557:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575462.805563:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575462.805566:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575462.805570:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575462.805573:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575462.805575:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575462.805577:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575462.805581:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575462.805583:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575462.805586:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575462.805589:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575462.805591:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575462.805594:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575462.805596:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575462.805598:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575462.805600:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575462.805602:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575462.805605:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575462.805608:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575462.805610:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575462.805613:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575462.805616:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575462.805619:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575462.805621:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575462.805624:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575462.805633:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575462.805637:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575462.805641:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575462.805642:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575462.805645:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575462.805648:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575462.805651:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575462.805658:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788575462.805664:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575462.805672:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575462.805681:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575462.805685:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575462.805690:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575462.805709:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575462.805740:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575462.805741:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:3.0:1788575462.805746:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575462.805747:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575462.805749:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1788575462.805751:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00080000:3.0:1788575462.805752:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575462.805757:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:2.0:1788575462.805761:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575462.805762:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12393
00000400:00000200:3.0:1788575462.805770:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575462.805775:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575462.805779:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575462.805780:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:2.0:1788575462.805785:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:3.0:1788575462.805785:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
02000000:00000001:2.0:1788575462.805790:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1788575462.805791:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575462.805795:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575462.805802:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393211264:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575462.805802:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575462.805808:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575462.805809:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575462.805812:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575462.805814:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:3.0:1788575462.805816:0:186668:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1788575462.805818:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575462.805820:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
02000000:00000001:2.0:1788575462.805823:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:3.0:1788575462.805824:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575462.805828:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
02000000:00000010:2.0:1788575462.805831:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
00000020:00000001:3.0:1788575462.805831:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575462.805835:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1788575462.805836:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000400:00000200:2.0:1788575462.805844:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575462.805844:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575462.805849:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575462.805852:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000040:3.0:1788575462.805852:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575462.805856:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575462.805859:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000400:00000010:2.0:1788575462.805861:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00080000:3.0:1788575462.805862:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575462.805867:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575462.805870:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393211264, portal 25
00000100:00000001:2.0:1788575462.805874:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1788575462.805875:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575462.805876:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
02000000:00000001:3.0:1788575462.805882:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575462.805884:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575462.805886:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575462.805890:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575462.805895:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788575462.805901:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00000001:3.0:1788575462.805901:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575462.805906:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575462.805907:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000010:3.0:1788575462.805911:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000400:00000010:2.0:1788575462.805912:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000020:00000040:3.0:1788575462.805916:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000200:2.0:1788575462.805918:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393211264, offset 0
02000000:00000001:3.0:1788575462.805920:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575462.805922:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575462.805925:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575462.805928:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575462.805928:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575462.805930:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575462.805936:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000400:00000200:2.0:1788575462.805940:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575462.805941:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575462.805949:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575462.805950:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575462.805960:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788575462.805961:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575462.805965:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575462.805966:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575462.805973:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1788575462.805973:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575462.805978:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575462.805986:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330906:330906:256:4294967295] 192.168.202.110@tcp LPNI seq info [330863:330863:8:4294967295]
00000100:00000001:3.0:1788575462.805991:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575462.805995:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575462.805996:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575462.806001:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575462.806003:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575462.806005:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575462.806010:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575462.806012:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1788575462.806014:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12393
00000400:00000200:3.0:1788575462.806020:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:2.0:1788575462.806025:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575462.806025:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575462.806029:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575462.806032:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575462.806033:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000100:00000001:3.0:1788575462.806034:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575462.806037:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575462.806041:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575462.806044:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.0:1788575462.806046:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1788575462.806047:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575462.806051:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.0:1788575462.806052:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000040:3.0:1788575462.806054:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575462.806059:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.1:1788575462.806063:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788575462.806064:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.1:1788575462.806070:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000040:3.0:1788575462.806071:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575462.806077:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000100:2.0:1788575462.806080:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575462.806082:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575462.806085:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000100:00000001:3.0:1788575462.806086:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575462.806091:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575462.806094:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575462.806095:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1788575462.806097:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575462.806101:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575462.806105:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:3.0:1788575462.806106:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575462.806110:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575462.806112:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000100:00000001:3.0:1788575462.806112:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575462.806115:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000010:2.0:1788575462.806117:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000008:00000001:3.0:1788575462.806119:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575462.806122:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.806124:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000010:3.0:1788575462.806126:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000100:00000200:2.0:1788575462.806132:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575462.806132:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575462.806137:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575462.806139:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575462.806143:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575462.806146:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575462.806150:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.806152:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000010:3.0:1788575462.806155:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000040:2.0:1788575462.806157:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575462.806160:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575462.806169:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575462.806171:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.806175:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575462.806181:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575462.806184:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575462.806188:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806193:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575462.806200:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575462.806203:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575462.806205:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575462.806209:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307067321
00000100:00000010:2.0:1788575462.806215:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c780.
00000100:00000040:3.0:1788575462.806221:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575462.806224:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:3.0:1788575462.806227:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12404)
00000100:00000001:2.0:1788575462.806232:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307067321
00000100:00000010:2.0:1788575462.806238:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c120.
00000100:00000001:2.0:1788575462.806241:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575462.806246:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575462.806249:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.806251:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575462.806257:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575462/real 1788575462]  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575462.806266:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575462.806268:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575462.806270:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575462.806273:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575462.806274:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575462.806276:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575462.806279:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575462.806287:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806294:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575462.806295:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806298:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575462.806300:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575462.806304:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806310:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575462.806314:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806329:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575462.806332:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806334:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575462.806335:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806337:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575462.806339:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575462.806342:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575462.806345:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575462.806348:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575462.806353:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575462.806360:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393211264:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575462.806365:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806367:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575462.806370:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393211264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806376:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575462.806378:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575462.806381:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575462.806419:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575462.806421:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575462.806423:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575462.806425:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575462.806428:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575462.806431:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575462.806433:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575462.806434:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575462.806438:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575462.806439:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806441:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806448:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575462.806450:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.806451:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575462.806453:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575462.806454:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575462.806459:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806465:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575462.806467:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575462.806468:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575462.806470:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575462.806472:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575462.806476:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211392:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575462.806479:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575462.806481:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575462.806482:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575462.806484:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575462.806487:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575462.806489:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575462.806492:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575462.806494:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575462.806498:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575462.806500:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393211392, portal 10
00000100:00000001:2.0:1788575462.806502:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575462.806504:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:2.0:1788575462.806510:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806516:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575462.806518:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575462.806520:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575462.806523:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393211392, offset 0
00000400:00000200:2.0:1788575462.806527:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575462.806530:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575462.806534:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575462.806536:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575462.806539:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575462.806541:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575462.806543:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575462.806546:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330907:330907:256:4294967295] 192.168.202.110@tcp LPNI seq info [330864:330864:8:4294967295]
00000400:00000200:2.0:1788575462.806549:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575462.806555:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575462.806559:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575462.806562:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575462.806566:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575462.806568:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788575462.806572:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575462.806575:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:2.0:1788575462.806578:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575462.806580:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575462.806583:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575462.806587:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575462.806589:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575462.806591:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575462.806594:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575462.806597:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806605:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575462.806608:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806614:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.806616:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575462.806619:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575462.806623:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575462.806626:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575462.806628:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307066407
00000100:00000001:2.0:1788575462.806631:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307067321
00000100:00000001:2.0:1788575462.806633:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307066407
00000100:00000001:2.0:1788575462.806635:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307067321
00000100:00000001:2.0:1788575462.806637:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575462.806639:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.806641:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575462.806645:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575462/real 1788575462]  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575462.806652:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575462.806653:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575462.806654:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575462.806657:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575462.806659:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575462.806661:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575462.806664:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575462.806671:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806677:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575462.806678:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806680:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575462.806681:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575462.806685:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806690:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575462.806694:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806700:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575462.806702:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806704:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575462.806705:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806707:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575462.806709:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575462.806725:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575462.806728:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575462.806730:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575462.806734:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575462.806741:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211392:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575462.806745:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575462.806748:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806759:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575462.806760:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575462.806762:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575462.806764:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575462.806766:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575462.806769:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211520:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575462.806772:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575462.806773:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575462.806775:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575462.806776:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575462.806780:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575462.806782:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575462.806785:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575462.806787:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575462.806791:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575462.806793:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393211520, portal 4
00000100:00000001:2.0:1788575462.806795:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575462.806796:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:2.0:1788575462.806801:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806808:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575462.806810:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575462.806812:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575462.806814:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393211520, offset 0
00000400:00000200:2.0:1788575462.806818:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575462.806821:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575462.806824:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575462.806827:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575462.806829:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575462.806831:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575462.806833:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575462.806836:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330908:330908:256:4294967295] 192.168.202.110@tcp LPNI seq info [330865:330865:8:4294967295]
00000400:00000200:2.0:1788575462.806840:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575462.806845:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575462.806849:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575462.806852:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575462.806855:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575462.806858:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1788575462.806861:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575462.806863:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:2.0:1788575462.806867:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575462.806868:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575462.806871:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575462.806874:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575462.806877:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575462.806879:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575462.806881:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575462.806884:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806891:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575462.806894:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806901:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.806902:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575462.806904:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575462.806907:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575462.806909:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575462.806911:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307066553
00000100:00000001:2.0:1788575462.806913:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307067321
00000100:00000001:2.0:1788575462.806915:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307066553
00000100:00000001:2.0:1788575462.806917:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307067321
00000100:00000001:2.0:1788575462.806919:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575462.806921:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.806923:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575462.806926:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575462/real 1788575462]  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575462.806933:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575462.806934:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575462.806936:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575462.806938:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575462.806940:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575462.806942:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575462.806945:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575462.806953:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806959:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575462.806960:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806962:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575462.806963:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575462.806966:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806973:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575462.806977:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.806984:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575462.806986:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806988:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575462.806989:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.806990:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575462.806992:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575462.806994:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575462.806997:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575462.806999:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575462.807003:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575462.807010:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211520:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575462.807014:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.807016:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575462.807018:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393211392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.807025:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575462.807026:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575462.807028:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575462.807031:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575462.807032:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575462.807033:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575462.807036:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575462.807037:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575462.807039:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575462.807041:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575462.807042:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788575462.807044:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575462.807045:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.807046:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575462.807050:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393211520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575462.807056:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575462.807057:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575462.807059:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575462.807061:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575462.807062:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575462.807063:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575462.807065:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575462.807067:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575462.807069:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575462.807070:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575462.807071:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788575462.807074:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575462.807075:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.807076:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575462.807079:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575462.807080:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575462.807082:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575462.807084:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575463.189537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575463.189544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575463.189548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575463.189549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575463.189552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575463.189552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575463.189612:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575463.189615:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575463.189618:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575463.317494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575463.317503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575463.317512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575463.317518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575463.317522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575463.317525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575463.317528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575463.317531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575463.317535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575463.765467:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307067321, at 4307067328
00000100:00000040:1.0:1788575463.765467:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307067321, at 4307067328
00000100:00000010:0.0:1788575463.765480:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000100:00000010:1.0:1788575463.765485:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00010000:00000001:3.0:1788575464.213505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575464.213511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575464.213516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575464.213518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575464.213523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575464.213524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575464.213547:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575464.213550:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575464.213553:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575464.341487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575464.341494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575464.341496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575464.341502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575464.341506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575464.341510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575464.341512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575464.341514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575464.341518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575465.237506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575465.237512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575465.237517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575465.237520:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575465.237522:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575465.237525:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575465.237528:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575465.237529:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575465.237531:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575465.237532:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575465.237535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575465.237536:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575465.237536:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575465.237537:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575465.237538:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575465.237539:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575465.237540:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575465.237542:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575465.237618:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575465.237623:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575465.237626:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575465.365547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575465.365557:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575465.365560:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575465.365567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575465.365571:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575465.365576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575465.365578:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575465.365581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575465.365584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575466.261527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575466.261534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575466.261538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575466.261539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575466.261542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575466.261542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575466.261602:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575466.261605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575466.261608:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575466.389554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575466.389564:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575466.389567:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575466.389574:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575466.389578:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575466.389583:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575466.389586:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575466.389590:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575466.389593:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575467.285463:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575467.285469:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575467.285471:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575467.285474:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575467.285478:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575467.285479:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575467.285481:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575467.285482:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575467.285485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575467.285486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575467.285489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575467.285489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575467.285535:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575467.285538:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575467.285540:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575467.413512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575467.413520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575467.413522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575467.413528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575467.413531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575467.413533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575467.413535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575467.413537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575467.413539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575467.925507:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575467.925515:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575467.925518:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575467.925528:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575467.925533:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575467.925539:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12399
00000400:00000200:1.0:1788575467.925548:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575467.925554:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575467.925558:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575467.925561:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925564:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575467.925566:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925567:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575467.925571:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575467.925573:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575467.925574:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575467.925577:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575467.925579:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575467.925581:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575467.925582:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925583:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575467.925584:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925586:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575467.925588:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575467.925590:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925592:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575467.925594:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575467.925596:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575467.925599:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925601:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575467.925603:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575467.925613:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788575467.925617:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575467.925620:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575467.925622:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925625:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575467.925628:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575467.925632:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575467.925639:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788575467.925645:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575467.925653:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575467.925660:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575467.925663:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575467.925666:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575467.925684:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575467.925686:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575467.925689:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575467.925690:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575467.925691:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575467.925693:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575467.925695:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12399
00000400:00000200:1.0:1788575467.925698:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575467.925700:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575467.925701:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575467.925704:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925705:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575467.925707:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925709:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575467.925710:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575467.925711:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575467.925712:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575467.925713:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575467.925714:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575467.925716:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575467.925717:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925719:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575467.925720:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925721:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575467.925723:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575467.925724:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1788575467.925725:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925729:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575467.925729:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
02000000:00000001:1.0:1788575467.925733:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575467.925738:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575467.925739:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1788575467.925740:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925742:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575467.925746:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000040:2.0:1788575467.925747:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1788575467.925750:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925754:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575467.925760:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788575467.925761:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575467.925763:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575467.925763:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925765:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1788575467.925765:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1788575467.925768:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575467.925768:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575467.925771:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575467.925773:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:1.0:1788575467.925775:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
02000000:00000001:2.0:1788575467.925776:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575467.925780:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925781:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575467.925788:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:2.0:1788575467.925789:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393211648:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575467.925794:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575467.925798:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575467.925801:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925804:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000001:2.0:1788575467.925805:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788575467.925807:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575467.925810:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
00000100:00000040:1.0:1788575467.925812:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575467.925814:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925827:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575467.925828:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575467.925833:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00080000:1.0:1788575467.925834:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575467.925839:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575467.925840:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575467.925841:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575467.925844:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575467.925847:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12399
00000400:00000200:1.0:1788575467.925853:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575467.925869:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575467.925871:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575467.925874:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925875:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575467.925876:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925877:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575467.925879:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575467.925880:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575467.925880:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575467.925882:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575467.925883:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575467.925884:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575467.925886:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925887:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575467.925888:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925889:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575467.925891:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575467.925892:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575467.925893:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575467.925894:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575467.925896:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575467.925898:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925899:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575467.925900:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575467.925901:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925902:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575467.925905:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788575467.925906:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575467.925908:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575467.925908:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575467.925909:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575467.925910:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575467.925912:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575467.925914:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788575467.925917:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575467.925920:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575467.925930:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575467.925931:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575467.925933:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575467.925938:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575467.925940:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575467.925942:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575467.925946:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12409)
00000400:00000010:2.0:1788575467.925949:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575467.925958:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393211648, portal 25
00000100:00000001:2.0:1788575467.925961:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575467.925963:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345470336 : -130483364081280 : ffff8953796f5180)
00000100:00000040:2.0:1788575467.925968:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.925974:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575467.925976:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575467.925979:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575467.925981:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393211648, offset 0
00000400:00000200:2.0:1788575467.925987:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575467.925995:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575467.926001:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575467.926006:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575467.926010:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575467.926013:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575467.926016:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575467.926020:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330909:330909:256:4294967295] 192.168.202.110@tcp LPNI seq info [330866:330866:8:4294967295]
00000400:00000200:2.0:1788575467.926025:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575467.926034:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575467.926039:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575467.926045:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575467.926052:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575467.926055:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788575467.926061:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575467.926064:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1788575467.926067:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575467.926069:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575467.926075:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575467.926079:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575467.926085:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575467.926087:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575467.926091:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575467.926095:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926104:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575467.926108:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575492 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575467.926116:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575467.926122:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575467.926126:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575467.926130:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307072441
00000100:00000010:2.0:1788575467.926135:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cde0.
00000100:00000001:2.0:1788575467.926141:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575467.926146:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307072441
00000100:00000010:2.0:1788575467.926149:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cb40.
00000100:00000001:2.0:1788575467.926152:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575467.926155:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575467.926158:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575467.926160:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575467.926165:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575467/real 1788575467]  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575467.926174:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575467.926176:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575467.926178:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575467.926180:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575467.926181:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575467.926182:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575467.926185:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575467.926193:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926200:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575467.926201:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926203:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575467.926205:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575467.926209:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926215:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575467.926219:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926224:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575467.926226:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926227:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575467.926227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926229:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575467.926231:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575467.926233:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575467.926236:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575467.926239:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575467.926243:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575467.926250:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393211648:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575467.926254:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926255:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575467.926257:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5180 x1875444393211648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926262:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575467.926265:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575467.926267:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575467.926270:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575467.926272:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575467.926273:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575467.926275:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575467.926278:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:2.0:1788575467.926281:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575467.926283:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575467.926284:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5180.
00000100:00000001:2.0:1788575467.926287:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575467.926288:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926290:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926297:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575467.926299:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575467.926301:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575467.926302:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575467.926304:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575467.926308:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926314:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575467.926316:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575467.926317:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575467.926319:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575467.926321:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575467.926324:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211776:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575467.926327:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575467.926329:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575467.926331:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575467.926333:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575467.926335:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575467.926337:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575467.926341:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575467.926343:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575467.926345:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575467.926347:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393211776, portal 10
00000100:00000001:2.0:1788575467.926349:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575467.926350:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345476608 : -130483364075008 : ffff8953796f6a00)
00000100:00000040:2.0:1788575467.926354:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926358:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575467.926359:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575467.926361:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575467.926362:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393211776, offset 0
00000400:00000200:2.0:1788575467.926364:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575467.926367:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575467.926369:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575467.926371:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575467.926373:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575467.926374:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575467.926377:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575467.926379:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330910:330910:256:4294967295] 192.168.202.110@tcp LPNI seq info [330867:330867:8:4294967295]
00000400:00000200:2.0:1788575467.926383:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575467.926390:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575467.926535:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575467.926539:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575467.926542:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575467.926544:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788575467.926550:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575467.926552:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1788575467.926554:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575467.926555:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575467.926557:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575467.926559:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575467.926562:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575467.926564:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575467.926566:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575467.926570:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926581:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575467.926583:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575492 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926587:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575467.926588:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575467.926591:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575467.926594:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575467.926598:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575467.926600:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307071527
00000100:00000001:2.0:1788575467.926602:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307072441
00000100:00000001:2.0:1788575467.926604:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307071527
00000100:00000001:2.0:1788575467.926607:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307072441
00000100:00000001:2.0:1788575467.926609:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575467.926610:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575467.926611:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575467.926614:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575467/real 1788575467]  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575467.926619:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575467.926620:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575467.926621:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575467.926623:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575467.926624:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575467.926625:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575467.926627:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575467.926632:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926635:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575467.926636:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926638:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575467.926639:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575467.926641:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926644:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575467.926646:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926650:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575467.926651:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926652:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575467.926653:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926654:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575467.926655:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575467.926657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575467.926659:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575467.926661:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575467.926665:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575467.926672:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211776:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575467.926675:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575467.926678:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926682:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575467.926683:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575467.926684:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575467.926685:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575467.926687:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575467.926689:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575467.926691:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575467.926693:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575467.926695:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575467.926696:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575467.926699:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575467.926701:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575467.926704:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575467.926706:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575467.926710:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575467.926712:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393211904, portal 4
00000100:00000001:2.0:1788575467.926714:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575467.926716:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345468544 : -130483364083072 : ffff8953796f4a80)
00000100:00000040:2.0:1788575467.926720:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926724:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575467.926726:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575467.926727:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575467.926730:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393211904, offset 0
00000400:00000200:2.0:1788575467.926733:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575467.926737:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575467.926741:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575467.926744:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575467.926745:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575467.926747:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575467.926749:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575467.926752:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330911:330911:256:4294967295] 192.168.202.110@tcp LPNI seq info [330868:330868:8:4294967295]
00000400:00000200:2.0:1788575467.926756:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575467.926762:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575467.926765:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575467.926767:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575467.926770:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575467.926771:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1788575467.926779:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575467.926781:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1788575467.926785:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575467.926787:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575467.926789:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575467.926791:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575467.926794:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575467.926795:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575467.926797:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575467.926801:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926808:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575467.926811:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575492 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926814:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575467.926816:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575467.926818:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575467.926821:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575467.926824:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575467.926826:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307071673
00000100:00000001:2.0:1788575467.926828:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307072441
00000100:00000001:2.0:1788575467.926829:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307071673
00000100:00000001:2.0:1788575467.926830:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307072441
00000100:00000001:2.0:1788575467.926832:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575467.926833:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575467.926834:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575467.926836:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575467/real 1788575467]  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575467.926841:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575467.926842:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575467.926844:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575467.926846:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575467.926847:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575467.926850:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575467.926852:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575467.926867:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926874:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575467.926876:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926877:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575467.926879:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575467.926882:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926887:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575467.926890:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926897:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575467.926899:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926901:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575467.926901:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926902:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575467.926904:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575467.926905:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575467.926907:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575467.926909:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575467.926911:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575467.926924:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393211904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575467.926926:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926927:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575467.926929:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6a00 x1875444393211776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926933:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575467.926934:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575467.926936:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575467.926938:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575467.926940:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575467.926941:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575467.926943:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575467.926945:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:2.0:1788575467.926947:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575467.926948:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575467.926949:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6a00.
00000100:00000001:2.0:1788575467.926951:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575467.926952:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926953:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575467.926955:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4a80 x1875444393211904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575492 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575467.926961:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575467.926962:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575467.926963:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575467.926965:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575467.926967:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575467.926968:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575467.926970:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575467.926971:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:2.0:1788575467.926973:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575467.926974:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575467.926975:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4a80.
00000100:00000001:2.0:1788575467.926978:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575467.926979:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926981:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575467.926984:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575467.926986:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575467.926987:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575467.926989:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575468.309513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575468.309518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575468.309522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575468.309523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575468.309525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575468.309525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575468.309570:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575468.309573:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575468.309576:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575468.437456:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575468.437461:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575468.437463:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575468.437466:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575468.437468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575468.437471:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575468.437473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575468.437474:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575468.437476:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575468.885457:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307072441, at 4307072448
00000100:00000040:1.0:1788575468.885457:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307072441, at 4307072448
00000100:00000010:1.0:1788575468.885470:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000100:00000010:0.0:1788575468.885471:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00010000:00000001:3.0:1788575469.333462:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575469.333468:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575469.333471:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575469.333472:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575469.333475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575469.333476:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575469.333519:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575469.333524:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575469.333528:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575469.461544:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575469.461550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575469.461552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575469.461557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575469.461559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575469.461562:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575469.461568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575469.461570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575469.461572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575470.357508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575470.357515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575470.357521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575470.357522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575470.357526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575470.357527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575470.357580:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575470.357583:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575470.357586:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575470.485507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575470.485514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575470.485517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575470.485522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575470.485526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575470.485530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575470.485533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575470.485536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575470.485539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575471.381489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575471.381495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575471.381499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575471.381501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575471.381503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575471.381504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575471.381541:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575471.381544:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575471.381547:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575471.509480:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575471.509486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575471.509489:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575471.509495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575471.509499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575471.509502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575471.509505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575471.509508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575471.509511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575472.405529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575472.405535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575472.405539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575472.405540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575472.405543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575472.405544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575472.405605:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575472.405609:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575472.405614:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575472.533513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575472.533521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575472.533524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575472.533529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575472.533534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575472.533539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575472.533543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575472.533547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575472.533551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575473.045584:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575473.045596:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575473.045599:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575473.045602:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575473.045607:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575473.045614:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12404
00000400:00000200:1.0:1788575473.045623:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575473.045630:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575473.045634:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575473.045637:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.045641:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575473.045644:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.045648:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575473.045652:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575473.045654:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575473.045657:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575473.045661:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575473.045663:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575473.045665:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575473.045669:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.045671:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575473.045672:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.045674:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575473.045676:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575473.045679:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575473.045680:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575473.045683:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575473.045685:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575473.045687:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575473.045689:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575473.045691:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575473.045697:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788575473.045700:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575473.045703:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575473.045704:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575473.045707:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575473.045709:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575473.045712:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575473.045718:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788575473.045725:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575473.045734:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575473.045752:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575473.045768:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575473.045773:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575473.045793:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575473.045800:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575473.045804:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575473.045806:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575473.045808:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575473.045811:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575473.045814:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12404
00000400:00000200:1.0:1788575473.045819:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575473.045823:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575473.045826:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575473.045830:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.045833:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575473.045835:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.045838:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575473.045839:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1788575473.045841:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575473.045845:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575473.045846:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1788575473.045847:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575473.045850:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1788575473.045852:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575473.045858:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575473.045861:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788575473.045862:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575473.045870:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.045876:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575473.045881:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575473.045883:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575473.045886:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575473.045888:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788575473.045892:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575473.045894:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:1.0:1788575473.045899:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575473.045900:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788575473.045906:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575473.045907:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575473.045912:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1788575473.045916:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393212032:192.168.202.110@tcp:250:kworker.0
02000000:00000001:1.0:1788575473.045920:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575473.045922:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788575473.045927:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575473.045928:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1788575473.045932:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575473.045933:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575473.045937:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:1.0:1788575473.045938:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575473.045944:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575473.045944:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575473.045949:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575473.045950:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575473.045958:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000400:00000200:2.0:1788575473.045958:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:1.0:1788575473.045964:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:2.0:1788575473.045965:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
02000000:00000001:1.0:1788575473.045970:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575473.045973:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575473.045974:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000001:1.0:1788575473.045977:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575473.045980:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1788575473.045982:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393212032, portal 25
02000000:00000001:1.0:1788575473.045983:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575473.045989:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000010:1.0:1788575473.045992:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:2.0:1788575473.045992:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345466752 : -130483364084864 : ffff8953796f4380)
00000100:00000001:1.0:1788575473.046001:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575473.046017:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788575473.046025:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575473.046033:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1788575473.046041:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1788575473.046042:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575473.046047:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575473.046048:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575473.046055:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393212032, offset 0
00000100:00000040:1.0:1788575473.046056:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575473.046068:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575473.046073:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575473.046077:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575473.046081:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788575473.046082:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575473.046086:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575473.046087:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575473.046090:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575473.046092:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:1.0:1788575473.046095:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12404
00000400:00000200:1.0:1788575473.046099:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575473.046101:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575473.046103:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575473.046106:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.046109:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575473.046112:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:1.0:1788575473.046113:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.046118:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575473.046121:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1788575473.046122:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:1.0:1788575473.046123:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575473.046126:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575473.046128:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:2.0:1788575473.046130:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:1.0:1788575473.046131:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575473.046134:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575473.046135:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:1.0:1788575473.046139:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.046143:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575473.046147:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330912:330912:256:4294967295] 192.168.202.110@tcp LPNI seq info [330869:330869:8:4294967295]
00000100:00000040:1.0:1788575473.046148:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575473.046151:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575473.046154:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575473.046159:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575473.046163:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575473.046163:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575473.046166:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575473.046170:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575473.046172:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575473.046174:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575473.046177:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575473.046181:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575473.046184:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1788575473.046185:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575473.046191:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788575473.046194:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.0:1788575473.046197:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788575473.046198:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575473.046202:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575473.046203:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575473.046204:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:2.0:1788575473.046205:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
02000000:00000001:1.0:1788575473.046207:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575473.046213:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000800:00000200:2.0:1788575473.046216:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788575473.046217:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575473.046223:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00100000:1.0:1788575473.046225:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575473.046237:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:2.1:1788575473.046237:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788575473.046239:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575473.046244:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000040:1.0:1788575473.046246:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:2.0:1788575473.046256:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575473.046260:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575473.046261:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000100:00000040:1.0:1788575473.046263:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575473.046267:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575473.046269:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:1.0:1788575473.046272:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12414)
00000400:00000200:2.0:1788575473.046277:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575473.046284:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575473.046288:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575473.046293:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575473.046298:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046310:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575473.046314:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575498 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046323:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575473.046325:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575473.046330:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575473.046334:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575473.046337:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307077561
00000100:00000010:2.0:1788575473.046343:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c8a0.
00000100:00000001:2.0:1788575473.046350:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575473.046366:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307077561
00000100:00000010:2.0:1788575473.046370:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c960.
00000100:00000001:2.0:1788575473.046373:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575473.046376:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575473.046379:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575473.046382:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575473.046387:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575473/real 1788575473]  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575473.046433:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575473.046435:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575473.046437:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575473.046441:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575473.046443:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575473.046445:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575473.046450:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575473.046460:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046468:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575473.046470:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.046472:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575473.046483:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575473.046488:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046497:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575473.046501:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046510:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575473.046513:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.046515:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575473.046517:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.046519:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575473.046522:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575473.046525:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575473.046529:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575473.046531:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575473.046536:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575473.046545:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393212032:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575473.046549:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.046552:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575473.046556:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4380 x1875444393212032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046564:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575473.046567:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575473.046569:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575473.046573:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575473.046575:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575473.046577:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575473.046579:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575473.046582:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:2.0:1788575473.046586:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575473.046588:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575473.046589:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4380.
00000100:00000001:2.0:1788575473.046593:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575473.046594:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.046597:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.046604:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575473.046606:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575473.046608:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575473.046610:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575473.046612:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575473.046616:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046624:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575473.046625:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575473.046628:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575473.046630:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575473.046632:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575473.046635:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575473.046639:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575473.046641:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575473.046643:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575473.046645:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575473.046648:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575473.046651:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575473.046655:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575473.046658:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575473.046662:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575473.046665:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393212160, portal 10
00000100:00000001:2.0:1788575473.046667:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575473.046668:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345465856 : -130483364085760 : ffff8953796f4000)
00000100:00000040:2.0:1788575473.046675:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046682:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575473.046685:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575473.046687:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575473.046690:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393212160, offset 0
00000400:00000200:2.0:1788575473.046695:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575473.046699:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575473.046703:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575473.046706:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575473.046709:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575473.046712:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575473.046715:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575473.046726:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330913:330913:256:4294967295] 192.168.202.110@tcp LPNI seq info [330870:330870:8:4294967295]
00000400:00000200:2.0:1788575473.046732:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575473.046739:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575473.046744:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575473.046748:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575473.046752:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575473.046755:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575473.046760:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575473.046763:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575473.046768:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575473.046769:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575473.046773:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575473.046777:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575473.046780:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575473.046782:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575473.046785:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575473.046789:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046798:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575473.046801:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575498 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046808:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575473.046810:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575473.046814:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575473.046818:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575473.046822:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575473.046825:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307076647
00000100:00000001:2.0:1788575473.046828:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307077561
00000100:00000001:2.0:1788575473.046830:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307076647
00000100:00000001:2.0:1788575473.046833:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307077561
00000100:00000001:2.0:1788575473.046835:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575473.046837:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575473.046839:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575473.046844:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575473/real 1788575473]  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575473.046853:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575473.046854:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575473.046856:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575473.046859:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575473.046861:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575473.046863:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575473.046866:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575473.046876:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046883:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575473.046884:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.046886:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575473.046888:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575473.046892:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046899:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575473.046903:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046910:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575473.046913:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.046915:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575473.046916:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.046918:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575473.046921:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575473.046924:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575473.046927:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575473.046930:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575473.046934:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575473.046943:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575473.046946:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575473.046950:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.046958:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575473.046960:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575473.046962:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575473.046964:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575473.046967:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575473.046970:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212288:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575473.046974:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575473.046976:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575473.046977:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575473.046979:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575473.046982:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575473.046985:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575473.046988:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575473.046991:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575473.046994:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575473.046997:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393212288, portal 4
00000100:00000001:2.0:1788575473.046999:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575473.047001:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345473920 : -130483364077696 : ffff8953796f5f80)
00000100:00000040:2.0:1788575473.047023:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.047031:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575473.047033:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575473.047036:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575473.047039:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393212288, offset 0
00000400:00000200:2.0:1788575473.047043:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575473.047048:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575473.047051:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575473.047054:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575473.047057:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575473.047060:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575473.047063:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575473.047066:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330914:330914:256:4294967295] 192.168.202.110@tcp LPNI seq info [330871:330871:8:4294967295]
00000400:00000200:2.0:1788575473.047071:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575473.047078:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575473.047083:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575473.047087:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575473.047092:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575473.047095:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575473.047099:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575473.047102:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575473.047106:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575473.047108:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575473.047112:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575473.047115:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575473.047119:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575473.047121:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575473.047123:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575473.047127:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.047136:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575473.047139:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575498 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.047153:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575473.047155:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575473.047158:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575473.047161:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575473.047164:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575473.047167:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307076793
00000100:00000001:2.0:1788575473.047169:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307077561
00000100:00000001:2.0:1788575473.047171:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307076793
00000100:00000001:2.0:1788575473.047174:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307077561
00000100:00000001:2.0:1788575473.047176:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575473.047178:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575473.047180:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575473.047184:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575473/real 1788575473]  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575473.047192:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575473.047193:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575473.047195:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575473.047198:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575473.047199:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575473.047202:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575473.047205:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575473.047214:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.047223:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575473.047224:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.047226:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575473.047228:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575473.047232:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.047240:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575473.047244:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.047252:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575473.047255:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.047256:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575473.047258:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.047260:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575473.047261:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575473.047264:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575473.047267:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575473.047270:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575473.047274:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575473.047283:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212288:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575473.047287:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.047289:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575473.047292:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4000 x1875444393212160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.047299:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575473.047301:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575473.047303:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575473.047305:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575473.047307:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575473.047309:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575473.047311:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575473.047313:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:2.0:1788575473.047316:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575473.047317:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575473.047319:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4000.
00000100:00000001:2.0:1788575473.047322:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575473.047323:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.047325:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575473.047328:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5f80 x1875444393212288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575498 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575473.047335:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575473.047337:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575473.047338:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575473.047340:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575473.047341:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575473.047343:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575473.047345:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575473.047346:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:2.0:1788575473.047348:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575473.047350:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575473.047351:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5f80.
00000100:00000001:2.0:1788575473.047353:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575473.047354:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.047356:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.047360:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575473.047361:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575473.047364:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575473.047366:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575473.429551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575473.429559:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575473.429565:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575473.429566:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575473.429570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575473.429571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575473.429665:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575473.429673:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575473.429678:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575473.557513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575473.557520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575473.557521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575473.557525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575473.557528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575473.557531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.557533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575473.557535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575473.557538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575474.005516:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307077561, at 4307077568
00000100:00000040:1.0:1788575474.005516:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307077561, at 4307077568
00000100:00000010:1.0:1788575474.005531:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00000100:00000010:0.0:1788575474.005536:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00010000:00000001:3.0:1788575474.453498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575474.453505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575474.453511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575474.453512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575474.453515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575474.453516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575474.453599:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575474.453607:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575474.453612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575474.581503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575474.581509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575474.581511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575474.581515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575474.581518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575474.581521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575474.581523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575474.581526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575474.581528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575475.477565:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575475.477574:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575475.477582:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575475.477586:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575475.477589:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575475.477593:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575475.477597:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575475.477600:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575475.477602:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575475.477605:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575475.477609:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575475.477611:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575475.477612:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575475.477613:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575475.477615:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575475.477617:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575475.477618:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575475.477620:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575475.477681:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575475.477686:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575475.477690:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575475.605554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575475.605560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575475.605562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575475.605566:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575475.605569:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575475.605572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575475.605574:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575475.605576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575475.605578:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575476.501557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575476.501567:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575476.501575:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575476.501577:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575476.501580:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575476.501582:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575476.501676:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575476.501681:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575476.501684:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575476.629504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575476.629510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575476.629511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575476.629515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575476.629518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575476.629521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575476.629523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575476.629525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575476.629527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575477.525548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575477.525558:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575477.525561:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575477.525565:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575477.525572:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575477.525575:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575477.525578:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575477.525581:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575477.525585:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575477.525586:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575477.525590:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575477.525591:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575477.525686:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575477.525693:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575477.525697:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575477.653546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575477.653553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575477.653555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575477.653560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575477.653564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575477.653568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575477.653571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575477.653573:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575477.653576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575478.165534:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575478.165544:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575478.165547:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575478.165551:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575478.165555:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575478.165562:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12409
00000400:00000200:1.0:1788575478.165571:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575478.165576:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575478.165579:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575478.165582:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.165585:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575478.165586:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.165588:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575478.165592:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575478.165593:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575478.165596:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575478.165598:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575478.165599:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575478.165601:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575478.165602:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.165604:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575478.165605:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.165606:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575478.165609:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575478.165611:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575478.165613:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575478.165615:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575478.165619:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575478.165621:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575478.165622:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575478.165624:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575478.165634:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788575478.165637:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575478.165640:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575478.165641:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575478.165644:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575478.165646:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575478.165650:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575478.165658:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788575478.165664:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575478.165671:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575478.165680:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575478.165683:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575478.165687:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575478.165707:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575478.165709:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575478.165712:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575478.165713:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575478.165714:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575478.165717:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575478.165719:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12409
00000400:00000200:1.0:1788575478.165722:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575478.165725:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575478.165727:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575478.165731:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.165733:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575478.165736:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.165738:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575478.165741:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575478.165743:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575478.165744:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1788575478.165746:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575478.165752:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575478.165752:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575478.165755:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1788575478.165758:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575478.165765:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575478.165767:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575478.165777:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.165785:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575478.165791:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1788575478.165792:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575478.165795:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575478.165796:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575478.165803:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575478.165804:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575478.165810:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1788575478.165813:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575478.165818:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575478.165818:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575478.165827:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1788575478.165830:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393212416:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575478.165835:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788575478.165836:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575478.165840:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575478.165841:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1788575478.165849:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575478.165849:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575478.165854:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:1.0:1788575478.165856:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575478.165859:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575478.165860:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575478.165864:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788575478.165866:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575478.165867:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575478.165869:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575478.165869:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
00000100:00000001:1.0:1788575478.165871:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575478.165872:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575478.165875:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575478.165878:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575478.165882:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000400:00000200:2.0:1788575478.165882:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575478.165889:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575478.165889:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575478.165906:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00100000:1.0:1788575478.165907:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788575478.165914:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393212416, portal 25
00000100:00000001:1.0:1788575478.165915:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575478.165920:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788575478.165921:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575478.165924:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
00000100:00000040:1.0:1788575478.165929:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575478.165937:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575478.165939:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575478.165944:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575478.165957:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575478.165962:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575478.165966:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575478.165971:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575478.165977:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12409
00000100:00000001:2.0:1788575478.165985:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575478.165992:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788575478.165995:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575478.165999:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000400:00000200:1.0:1788575478.166000:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575478.166004:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575478.166010:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575478.166011:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393212416, offset 0
00000100:00000001:1.0:1788575478.166012:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575478.166014:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.166018:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575478.166021:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575478.166022:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575478.166023:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575478.166025:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575478.166026:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575478.166033:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:1.0:1788575478.166034:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575478.166042:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.166044:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575478.166046:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.166047:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575478.166050:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575478.166052:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788575478.166056:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575478.166060:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575478.166062:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575478.166064:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575478.166066:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575478.166069:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:1.0:1788575478.166070:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575478.166074:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575478.166080:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575478.166082:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575478.166083:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000010:1.0:1788575478.166085:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788575478.166088:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575478.166093:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575478.166093:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:1.0:1788575478.166099:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575478.166104:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575478.166106:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1788575478.166107:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575478.166109:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575478.166114:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000400:00000200:2.0:1788575478.166116:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:1.0:1788575478.166118:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575478.166127:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575478.166131:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330915:330915:256:4294967295] 192.168.202.110@tcp LPNI seq info [330872:330872:8:4294967295]
00000100:00000001:1.0:1788575478.166138:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575478.166141:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575478.166165:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788575478.166171:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575478.166194:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1788575478.166196:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575478.166203:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575478.166210:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:2.0:1788575478.166216:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788575478.166218:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12419)
00000800:00000010:2.0:1788575478.166223:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575478.166233:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575478.166237:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575478.166244:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575478.166247:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575478.166251:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575478.166253:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575478.166259:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575478.166263:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575478.166269:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575478.166271:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575478.166282:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575478.166290:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166299:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575478.166301:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575503 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166306:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575478.166307:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575478.166313:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575478.166316:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575478.166318:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307082681
00000100:00000010:2.0:1788575478.166322:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd20.
00000100:00000001:2.0:1788575478.166327:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575478.166331:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307082681
00000100:00000010:2.0:1788575478.166333:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c360.
00000100:00000001:2.0:1788575478.166335:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575478.166337:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575478.166339:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575478.166341:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575478.166343:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575478/real 1788575478]  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575478.166349:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575478.166350:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575478.166352:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575478.166354:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575478.166355:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575478.166356:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575478.166358:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575478.166363:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166367:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575478.166368:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166370:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575478.166371:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575478.166373:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166377:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575478.166380:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166384:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575478.166386:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166387:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575478.166388:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166389:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575478.166416:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575478.166418:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575478.166421:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575478.166423:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575478.166425:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575478.166431:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393212416:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575478.166434:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166435:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575478.166437:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393212416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166440:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575478.166442:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575478.166444:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575478.166446:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575478.166448:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575478.166450:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575478.166451:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575478.166453:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:2.0:1788575478.166456:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575478.166457:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575478.166458:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:2.0:1788575478.166460:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575478.166461:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166462:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166469:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575478.166470:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575478.166471:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575478.166473:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575478.166474:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575478.166476:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575478.166482:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575478.166483:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575478.166484:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575478.166486:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575478.166487:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212544:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575478.166489:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575478.166491:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575478.166492:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575478.166493:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575478.166495:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575478.166496:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575478.166499:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575478.166501:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575478.166503:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575478.166505:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393212544, portal 10
00000100:00000001:2.0:1788575478.166506:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575478.166507:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000040:2.0:1788575478.166511:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166515:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575478.166516:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575478.166517:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575478.166519:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393212544, offset 0
00000400:00000200:2.0:1788575478.166521:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575478.166523:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575478.166526:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575478.166528:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575478.166530:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575478.166532:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575478.166533:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575478.166535:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330916:330916:256:4294967295] 192.168.202.110@tcp LPNI seq info [330873:330873:8:4294967295]
00000400:00000200:2.0:1788575478.166537:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575478.166541:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575478.166544:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575478.166546:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575478.166548:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575478.166554:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575478.166557:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575478.166559:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575478.166561:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575478.166562:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575478.166564:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575478.166566:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575478.166568:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575478.166569:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575478.166570:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575478.166572:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166577:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575478.166579:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575503 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166582:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575478.166583:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575478.166585:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575478.166587:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575478.166589:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575478.166590:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307081767
00000100:00000001:2.0:1788575478.166592:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307082681
00000100:00000001:2.0:1788575478.166594:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307081767
00000100:00000001:2.0:1788575478.166595:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307082681
00000100:00000001:2.0:1788575478.166596:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575478.166597:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575478.166599:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575478.166600:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575478/real 1788575478]  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575478.166604:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575478.166605:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575478.166606:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575478.166607:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575478.166608:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575478.166610:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575478.166611:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575478.166616:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166619:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575478.166620:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166621:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575478.166622:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575478.166623:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166627:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575478.166629:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166632:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575478.166634:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166635:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575478.166636:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166637:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575478.166638:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575478.166640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575478.166642:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575478.166643:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575478.166651:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575478.166656:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212544:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575478.166658:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575478.166660:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166664:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575478.166665:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575478.166667:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575478.166668:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575478.166669:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575478.166670:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212672:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575478.166672:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575478.166673:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575478.166674:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575478.166675:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575478.166677:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575478.166678:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575478.166680:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575478.166681:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575478.166683:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575478.166685:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393212672, portal 4
00000100:00000001:2.0:1788575478.166686:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575478.166687:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:2.0:1788575478.166690:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166694:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575478.166695:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575478.166696:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575478.166698:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393212672, offset 0
00000400:00000200:2.0:1788575478.166700:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575478.166702:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575478.166704:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575478.166706:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575478.166707:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575478.166709:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575478.166710:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575478.166712:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330917:330917:256:4294967295] 192.168.202.110@tcp LPNI seq info [330874:330874:8:4294967295]
00000400:00000200:2.0:1788575478.166714:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575478.166718:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575478.166720:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575478.166722:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575478.166724:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575478.166726:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575478.166728:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575478.166729:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575478.166732:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575478.166733:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575478.166734:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575478.166736:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575478.166738:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575478.166739:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575478.166740:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575478.166742:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166746:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575478.166748:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575503 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166752:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575478.166753:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575478.166754:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575478.166756:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575478.166758:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575478.166759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307081913
00000100:00000001:2.0:1788575478.166760:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307082681
00000100:00000001:2.0:1788575478.166762:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307081913
00000100:00000001:2.0:1788575478.166763:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307082681
00000100:00000001:2.0:1788575478.166764:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575478.166765:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575478.166766:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575478.166768:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575478/real 1788575478]  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575478.166772:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575478.166773:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575478.166773:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575478.166774:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575478.166775:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575478.166776:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575478.166778:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575478.166782:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166786:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575478.166787:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166788:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575478.166788:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575478.166791:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166795:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575478.166797:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166800:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575478.166801:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575478.166803:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166804:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575478.166805:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575478.166806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575478.166808:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575478.166809:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575478.166811:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575478.166815:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212672:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575478.166817:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166818:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575478.166819:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393212544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166823:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575478.166824:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575478.166825:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575478.166827:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575478.166827:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575478.166828:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575478.166829:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575478.166830:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:2.0:1788575478.166832:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575478.166833:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575478.166834:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:2.0:1788575478.166835:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575478.166836:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166837:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575478.166839:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393212672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575503 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575478.166842:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575478.166843:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575478.166844:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575478.166845:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575478.166845:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575478.166846:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575478.166847:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575478.166847:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:2.0:1788575478.166849:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575478.166849:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575478.166850:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:2.0:1788575478.166851:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575478.166852:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166852:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.166854:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575478.166855:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575478.166856:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575478.166857:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575478.549510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575478.549518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575478.549525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575478.549528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575478.549533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575478.549535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575478.549581:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575478.549585:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575478.549589:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575478.677464:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575478.677471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575478.677473:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575478.677478:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575478.677481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575478.677484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.677487:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575478.677489:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575478.677491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575479.125454:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307082681, at 4307082688
00000100:00000040:0.0:1788575479.125465:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307082681, at 4307082688
00000100:00000010:1.0:1788575479.125468:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000100:00000010:0.0:1788575479.125475:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00010000:00000001:3.0:1788575479.573468:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575479.573473:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575479.573476:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575479.573477:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575479.573480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575479.573481:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575479.573528:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575479.573531:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575479.573534:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575479.701484:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575479.701491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575479.701493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575479.701497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575479.701500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575479.701503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575479.701505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575479.701507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575479.701510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575480.597525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575480.597531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575480.597537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575480.597539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575480.597543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575480.597544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575480.597666:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575480.597669:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575480.597672:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575480.725554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575480.725561:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575480.725564:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575480.725570:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575480.725583:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575480.725588:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575480.725591:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575480.725593:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575480.725597:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575481.237480:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1788575481.237489:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1788575481.237494:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:1.0:1788575481.237661:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788575481.237666:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575481.237683:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788575481.237689:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575481.237692:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575481.621476:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575481.621481:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575481.621485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575481.621486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575481.621488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575481.621489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575481.621548:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575481.621551:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575481.621554:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575481.749494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575481.749502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575481.749504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575481.749509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575481.749513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575481.749517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575481.749520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575481.749523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575481.749526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575482.645495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575482.645501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575482.645505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575482.645506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575482.645511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575482.645513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575482.645572:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575482.645576:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575482.645579:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575482.773494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575482.773501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575482.773504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575482.773509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575482.773512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575482.773514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575482.773516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575482.773518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575482.773520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575483.285483:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575483.285492:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575483.285494:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575483.285497:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575483.285501:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575483.285506:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12414
00000400:00000200:1.0:1788575483.285513:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575483.285518:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575483.285521:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575483.285524:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285527:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575483.285529:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285532:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575483.285536:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575483.285538:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575483.285541:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575483.285544:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575483.285546:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575483.285548:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575483.285551:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285553:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575483.285554:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285556:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575483.285560:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575483.285563:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285565:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575483.285568:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575483.285571:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575483.285573:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575483.285575:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575483.285578:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575483.285586:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788575483.285593:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575483.285597:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575483.285598:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575483.285601:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575483.285604:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575483.285607:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575483.285612:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788575483.285617:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575483.285625:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575483.285631:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575483.285634:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575483.285638:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575483.285653:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575483.285656:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575483.285658:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575483.285659:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575483.285660:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575483.285662:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575483.285664:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12414
00000400:00000200:1.0:1788575483.285667:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575483.285669:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575483.285670:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575483.285672:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285674:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575483.285675:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285677:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575483.285678:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575483.285679:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575483.285680:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575483.285682:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575483.285683:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575483.285685:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575483.285686:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788575483.285687:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285690:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575483.285690:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1788575483.285692:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575483.285693:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1788575483.285694:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575483.285696:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575483.285698:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285702:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575483.285705:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575483.285706:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788575483.285709:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575483.285714:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575483.285718:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575483.285719:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:1.0:1788575483.285722:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788575483.285722:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1788575483.285727:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575483.285728:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788575483.285729:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575483.285731:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:1.0:1788575483.285733:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
02000000:00000001:2.0:1788575483.285734:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788575483.285735:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00100000:2.0:1788575483.285738:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393212800:192.168.202.110@tcp:250:kworker.0
02000000:00000001:1.0:1788575483.285739:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575483.285741:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575483.285744:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1788575483.285744:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788575483.285747:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575483.285749:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575483.285749:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:1.0:1788575483.285752:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
02000000:00000001:2.0:1788575483.285752:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575483.285754:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788575483.285756:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575483.285760:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
00000100:00100000:1.0:1788575483.285761:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575483.285763:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575483.285769:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575483.285770:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575483.285773:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575483.285775:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000040:1.0:1788575483.285778:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575483.285781:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000001:1.0:1788575483.285786:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575483.285789:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1788575483.285790:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393212800, portal 25
00000100:00080000:1.0:1788575483.285792:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575483.285793:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788575483.285794:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1788575483.285795:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00080000:1.0:1788575483.285796:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575483.285801:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:2.0:1788575483.285804:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788575483.285805:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12414
00000400:00000200:1.0:1788575483.285811:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575483.285817:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575483.285821:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575483.285821:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788575483.285826:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575483.285828:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285829:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575483.285831:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000040:1.0:1788575483.285833:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575483.285835:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393212800, offset 0
00000100:00000001:1.0:1788575483.285836:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575483.285840:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575483.285841:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575483.285842:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575483.285843:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:2.0:1788575483.285846:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:1.0:1788575483.285847:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575483.285850:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575483.285852:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285853:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575483.285856:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788575483.285857:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285859:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575483.285861:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575483.285863:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:1.0:1788575483.285865:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285868:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575483.285869:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:1.0:1788575483.285870:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575483.285873:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575483.285874:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:1.0:1788575483.285876:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575483.285879:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1788575483.285880:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575483.285883:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575483.285884:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000008:00000001:1.0:1788575483.285888:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575483.285891:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330918:330918:256:4294967295] 192.168.202.110@tcp LPNI seq info [330875:330875:8:4294967295]
00000100:00000001:1.0:1788575483.285892:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575483.285897:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000400:00000200:2.0:1788575483.285898:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:1.0:1788575483.285902:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575483.285904:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575483.285905:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575483.285909:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575483.285912:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575483.285912:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1788575483.285915:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575483.285919:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000800:00000200:2.0:1788575483.285919:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575483.285923:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575483.285925:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000100:00100000:1.0:1788575483.285928:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1788575483.285934:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788575483.285939:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:2.0:1788575483.285940:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1788575483.285941:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575483.285947:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788575483.285949:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575483.285954:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1788575483.285959:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1788575483.285959:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788575483.285962:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:2.0:1788575483.285963:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000100:00000040:1.0:1788575483.285968:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575483.285970:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:1.0:1788575483.285972:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12424)
00000400:00000200:2.0:1788575483.285976:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575483.285979:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575483.285981:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575483.285984:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575483.285988:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.285994:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575483.285996:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575508 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286001:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575483.286002:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575483.286005:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575483.286008:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575483.286010:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307087801
00000100:00000010:2.0:1788575483.286013:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c240.
00000100:00000001:2.0:1788575483.286018:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575483.286022:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307087801
00000100:00000010:2.0:1788575483.286024:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ce40.
00000100:00000001:2.0:1788575483.286026:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575483.286028:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575483.286029:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575483.286031:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575483.286033:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575483/real 1788575483]  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575483.286039:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575483.286040:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575483.286041:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575483.286043:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575483.286045:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575483.286047:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575483.286050:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575483.286057:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286063:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575483.286065:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286067:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575483.286070:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575483.286074:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286081:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575483.286085:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286091:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575483.286092:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286093:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575483.286094:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286096:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575483.286097:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575483.286099:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575483.286101:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575483.286103:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575483.286105:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575483.286110:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393212800:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575483.286113:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286114:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575483.286116:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393212800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286120:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575483.286122:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575483.286124:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575483.286126:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575483.286127:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575483.286127:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575483.286129:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575483.286131:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:2.0:1788575483.286133:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575483.286135:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575483.286136:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:2.0:1788575483.286138:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575483.286139:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286140:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286145:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575483.286146:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575483.286148:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575483.286148:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575483.286149:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575483.286151:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286156:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575483.286157:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575483.286158:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575483.286159:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575483.286160:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575483.286162:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575483.286163:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575483.286165:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575483.286166:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575483.286167:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575483.286169:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575483.286170:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575483.286172:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575483.286174:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575483.286176:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575483.286178:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393212928, portal 10
00000100:00000001:2.0:1788575483.286179:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575483.286180:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:2.0:1788575483.286184:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286188:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575483.286189:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575483.286191:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575483.286192:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393212928, offset 0
00000400:00000200:2.0:1788575483.286194:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575483.286197:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575483.286199:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575483.286201:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575483.286202:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575483.286204:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575483.286205:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575483.286207:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330919:330919:256:4294967295] 192.168.202.110@tcp LPNI seq info [330876:330876:8:4294967295]
00000400:00000200:2.0:1788575483.286210:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575483.286214:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575483.286217:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575483.286219:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575483.286221:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575483.286223:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575483.286225:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575483.286227:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575483.286229:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575483.286230:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575483.286232:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575483.286234:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575483.286235:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575483.286236:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575483.286238:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575483.286240:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286244:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575483.286246:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575508 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286254:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575483.286255:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575483.286258:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575483.286260:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575483.286262:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575483.286263:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307086886
00000100:00000001:2.0:1788575483.286265:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307087801
00000100:00000001:2.0:1788575483.286266:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307086886
00000100:00000001:2.0:1788575483.286268:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307087801
00000100:00000001:2.0:1788575483.286269:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575483.286270:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575483.286272:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575483.286274:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575483/real 1788575483]  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575483.286278:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575483.286279:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575483.286280:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575483.286281:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575483.286282:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575483.286283:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575483.286285:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575483.286289:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286293:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575483.286294:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286295:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575483.286315:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575483.286319:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286322:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575483.286324:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286329:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575483.286330:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286331:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575483.286332:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286333:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575483.286334:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575483.286335:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575483.286337:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575483.286338:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575483.286340:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575483.286344:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393212928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575483.286347:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575483.286349:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286352:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575483.286353:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575483.286354:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575483.286355:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575483.286356:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575483.286358:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575483.286360:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575483.286361:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575483.286362:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575483.286363:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575483.286365:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575483.286366:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575483.286368:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575483.286370:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575483.286372:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575483.286374:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393213056, portal 4
00000100:00000001:2.0:1788575483.286375:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575483.286376:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:2.0:1788575483.286380:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286383:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575483.286385:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575483.286386:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575483.286387:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393213056, offset 0
00000400:00000200:2.0:1788575483.286389:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575483.286416:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575483.286418:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575483.286420:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575483.286421:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575483.286423:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575483.286424:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575483.286426:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330920:330920:256:4294967295] 192.168.202.110@tcp LPNI seq info [330877:330877:8:4294967295]
00000400:00000200:2.0:1788575483.286428:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575483.286432:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575483.286434:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575483.286437:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575483.286439:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575483.286441:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575483.286443:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575483.286444:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575483.286446:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575483.286448:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575483.286449:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575483.286451:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575483.286453:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575483.286454:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575483.286456:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575483.286458:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286463:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575483.286465:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575508 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286469:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575483.286470:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575483.286471:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575483.286473:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575483.286475:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575483.286476:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307087033
00000100:00000001:2.0:1788575483.286478:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307087801
00000100:00000001:2.0:1788575483.286479:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307087033
00000100:00000001:2.0:1788575483.286480:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307087801
00000100:00000001:2.0:1788575483.286481:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575483.286482:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575483.286483:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575483.286485:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575483/real 1788575483]  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575483.286489:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575483.286490:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575483.286491:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575483.286492:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575483.286493:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575483.286494:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575483.286496:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575483.286501:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286508:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575483.286509:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286510:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575483.286511:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575483.286513:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286516:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575483.286518:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286522:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575483.286523:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575483.286524:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286525:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575483.286526:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575483.286528:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575483.286529:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575483.286530:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575483.286532:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575483.286536:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575483.286538:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286539:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575483.286541:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393212928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286545:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575483.286546:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575483.286547:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575483.286548:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575483.286549:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575483.286550:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575483.286551:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575483.286553:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:2.0:1788575483.286554:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575483.286555:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575483.286556:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:2.0:1788575483.286557:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575483.286558:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286559:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575483.286561:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393213056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575508 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575483.286564:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575483.286565:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575483.286566:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575483.286567:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575483.286567:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575483.286568:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575483.286569:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575483.286570:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:2.0:1788575483.286571:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575483.286571:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575483.286572:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:2.0:1788575483.286573:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575483.286574:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286575:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.286577:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575483.286577:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575483.286578:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575483.286579:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575483.669490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575483.669495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575483.669498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575483.669499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575483.669501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575483.669502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575483.669560:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575483.669564:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575483.669567:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575483.797460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575483.797465:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575483.797466:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575483.797470:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575483.797472:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575483.797475:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.797477:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575483.797479:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575483.797481:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575484.245499:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307087801, at 4307087808
00000100:00000040:1.0:1788575484.245499:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307087801, at 4307087808
00000100:00000010:0.0:1788575484.245515:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000100:00000010:1.0:1788575484.245517:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00010000:00000001:3.0:1788575484.693555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575484.693561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575484.693564:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575484.693577:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575484.693580:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575484.693581:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575484.693632:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575484.693635:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575484.693637:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575484.821516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575484.821523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575484.821525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575484.821530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575484.821532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575484.821535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575484.821537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575484.821539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575484.821541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575485.717509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575485.717515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575485.717521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575485.717525:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575485.717527:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575485.717531:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575485.717534:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575485.717537:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575485.717540:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575485.717542:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575485.717546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575485.717547:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575485.717549:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575485.717550:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575485.717552:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575485.717553:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575485.717555:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575485.717557:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575485.717595:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575485.717597:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575485.717599:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575485.845515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575485.845520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575485.845522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575485.845525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575485.845528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575485.845531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575485.845532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575485.845534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575485.845537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575486.741487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575486.741498:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575486.741500:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575486.741503:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575486.741508:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575486.741510:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575486.741512:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575486.741514:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575486.741518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575486.741520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575486.741524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575486.741526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575486.741593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575486.741598:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575486.741601:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575486.869592:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575486.869604:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575486.869606:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575486.869613:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575486.869620:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575486.869624:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575486.869628:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575486.869631:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575486.869635:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575487.765558:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575487.765566:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575487.765572:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575487.765573:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575487.765583:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575487.765585:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575487.765655:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575487.765660:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575487.765663:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575487.893507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575487.893514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575487.893516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575487.893522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575487.893526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575487.893529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575487.893532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575487.893534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575487.893537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575488.405702:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575488.405717:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575488.405720:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575488.405725:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575488.405730:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575488.405739:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12419
00000400:00000200:2.0:1788575488.405749:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575488.405756:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575488.405759:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575488.405762:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.405766:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575488.405769:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.405771:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575488.405775:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575488.405778:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575488.405780:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575488.405784:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575488.405787:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575488.405789:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575488.405792:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.405794:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575488.405796:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.405798:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575488.405801:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575488.405805:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575488.405807:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575488.405811:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575488.405814:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575488.405817:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575488.405820:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575488.405822:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575488.405833:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02680.
00000020:00000040:2.0:1788575488.405837:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575488.405842:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575488.405843:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575488.405847:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575488.405850:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575488.405854:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575488.405862:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788575488.405870:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575488.405880:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575488.405890:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575488.405895:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575488.405900:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000450d1599] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575488.405924:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575488.405933:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575488.405937:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575488.405942:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575488.405944:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575488.405947:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575488.405951:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12419
00000100:00000001:3.0:1788575488.405951:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575488.405957:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000400:00000200:2.0:1788575488.405958:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575488.405963:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000400:00000200:2.0:1788575488.405964:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575488.405968:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575488.405973:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575488.405974:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.405979:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575488.405984:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.405990:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575488.405992:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1788575488.405993:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1788575488.405995:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575488.405996:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575488.406000:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:3.0:1788575488.406001:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1788575488.406003:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
02000000:00000001:3.0:1788575488.406005:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1788575488.406007:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:3.0:1788575488.406010:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575488.406012:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00100000:3.0:1788575488.406017:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393213184:192.168.202.110@tcp:250:kworker.0
00000100:00000040:2.0:1788575488.406020:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.406025:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575488.406025:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:2.0:1788575488.406029:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.406032:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575488.406032:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575488.406036:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575488.406037:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575488.406041:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:2.0:1788575488.406042:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575488.406047:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575488.406048:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:2.0:1788575488.406049:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575488.406052:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575488.406058:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575488.406062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575488.406064:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575488.406069:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1788575488.406069:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000008:00100000:2.0:1788575488.406076:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1788575488.406079:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000008:00000001:2.0:1788575488.406080:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575488.406084:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575488.406089:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00700.
00000100:00000200:3.0:1788575488.406089:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393213184, portal 25
00000020:00000040:2.0:1788575488.406092:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575488.406095:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575488.406096:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575488.406098:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575488.406102:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1788575488.406102:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575488.406106:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
02000000:00000001:2.0:1788575488.406107:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575488.406114:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000100:00000040:3.0:1788575488.406117:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575488.406120:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575488.406129:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575488.406131:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575488.406138:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575488.406143:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000001:2.0:1788575488.406144:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575488.406148:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575488.406153:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e69941e] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575488.406155:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393213184, offset 0
00000400:00000200:3.0:1788575488.406169:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575488.406178:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575488.406181:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575488.406182:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575488.406188:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575488.406190:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575488.406193:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575488.406194:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1788575488.406200:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575488.406205:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12419
00000400:00000200:3.0:1788575488.406206:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575488.406214:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575488.406218:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575488.406221:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575488.406225:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330921:330921:256:4294967295] 192.168.202.110@tcp LPNI seq info [330878:330878:8:4294967295]
00000400:00000200:3.0:1788575488.406230:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575488.406236:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575488.406243:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1788575488.406247:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1788575488.406249:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575488.406257:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.406261:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575488.406262:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575488.406266:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575488.406269:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000010:3.0:1788575488.406271:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000020:00000001:2.0:1788575488.406272:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575488.406276:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575488.406278:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575488.406281:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:3.0:1788575488.406282:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1788575488.406285:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.0:1788575488.406288:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1788575488.406289:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575488.406297:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575488.406301:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575488.406302:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575488.406306:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575488.406308:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1788575488.406309:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575488.406314:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575488.406319:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000100:3.0:1788575488.406319:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575488.406324:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000100:00000001:2.0:1788575488.406325:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575488.406329:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575488.406332:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575488.406334:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1788575488.406335:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575488.406339:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575488.406343:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000008:00100000:2.0:1788575488.406344:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575488.406349:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575488.406351:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575488.406352:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000100:00000010:2.0:1788575488.406356:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000400:00000010:3.0:1788575488.406356:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000020:00000040:2.0:1788575488.406362:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:3.0:1788575488.406363:0:2361:0:(events.c:35:request_out_callback()) Process entered
02000000:00000001:2.0:1788575488.406365:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575488.406369:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575488.406370:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575488.406373:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575488.406376:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575488.406379:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.406387:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000010:2.0:1788575488.406389:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11400.
00000100:00000040:3.0:1788575488.406412:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575513 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406421:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.406423:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575488.406429:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.406430:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575488.406436:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575488.406438:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575488.406456:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307092922
00000100:00000010:3.0:1788575488.406465:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:2.0:1788575488.406466:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575488.406471:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.406476:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:2.0:1788575488.406477:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575488.406487:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575488.406489:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575488.406492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307092922
00000100:00000040:2.0:1788575488.406494:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:3.0:1788575488.406498:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000040:2.0:1788575488.406501:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12429)
00000100:00000001:3.0:1788575488.406503:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575488.406509:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575488.406512:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.406515:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575488.406521:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575488/real 1788575488]  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575488.406531:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575488.406533:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575488.406536:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575488.406539:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575488.406540:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575488.406543:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575488.406546:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575488.406554:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406560:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575488.406562:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.406565:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575488.406567:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575488.406571:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406580:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575488.406585:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406592:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575488.406595:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.406597:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575488.406598:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.406601:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575488.406603:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575488.406606:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575488.406610:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575488.406612:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575488.406617:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575488.406625:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393213184:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575488.406630:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.406632:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575488.406635:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393213184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406641:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575488.406644:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575488.406647:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575488.406650:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575488.406652:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575488.406653:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575488.406656:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575488.406658:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575488.406662:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575488.406665:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575488.406666:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575488.406670:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575488.406671:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.406674:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.406682:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575488.406684:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.406686:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575488.406688:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575488.406689:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575488.406693:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406700:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575488.406701:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575488.406703:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575488.406705:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575488.406707:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575488.406710:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213312:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575488.406713:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575488.406715:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575488.406717:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575488.406718:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575488.406721:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575488.406723:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575488.406726:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575488.406729:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575488.406733:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575488.406736:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393213312, portal 10
00000100:00000001:3.0:1788575488.406737:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575488.406739:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:3.0:1788575488.406745:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406758:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575488.406761:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575488.406763:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575488.406765:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393213312, offset 0
00000400:00000200:3.0:1788575488.406769:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575488.406773:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575488.406777:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575488.406780:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575488.406782:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575488.406785:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575488.406787:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575488.406790:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330922:330922:256:4294967295] 192.168.202.110@tcp LPNI seq info [330879:330879:8:4294967295]
00000400:00000200:3.0:1788575488.406793:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575488.406800:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575488.406804:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575488.406807:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575488.406810:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575488.406813:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575488.406817:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575488.406820:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575488.406824:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575488.406826:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575488.406829:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575488.406832:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575488.406834:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575488.406836:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575488.406839:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575488.406842:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406849:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575488.406852:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575513 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406859:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.406860:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575488.406864:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575488.406868:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575488.406871:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575488.406873:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307092007
00000100:00000001:3.0:1788575488.406876:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307092922
00000100:00000001:3.0:1788575488.406878:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307092007
00000100:00000001:3.0:1788575488.406880:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307092922
00000100:00000001:3.0:1788575488.406882:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575488.406884:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.406886:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575488.406889:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575488/real 1788575488]  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575488.406896:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575488.406898:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575488.406899:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575488.406902:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575488.406903:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575488.406905:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575488.406908:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575488.406916:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406922:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575488.406924:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.406925:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575488.406927:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575488.406930:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406937:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575488.406940:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406947:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575488.406949:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.406951:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575488.406952:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.406954:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575488.406956:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575488.406959:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575488.406962:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575488.406964:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575488.406968:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575488.406975:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213312:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575488.406979:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575488.406982:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.406988:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575488.406989:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575488.406991:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575488.406993:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575488.406995:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575488.406997:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213440:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575488.407000:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575488.407002:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575488.407003:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575488.407005:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575488.407008:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575488.407010:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575488.407013:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575488.407016:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575488.407019:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575488.407022:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393213440, portal 4
00000100:00000001:3.0:1788575488.407023:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575488.407025:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575488.407031:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.407036:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575488.407038:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575488.407040:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575488.407043:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393213440, offset 0
00000400:00000200:3.0:1788575488.407046:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575488.407049:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575488.407053:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575488.407055:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575488.407058:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575488.407060:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575488.407062:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575488.407065:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330923:330923:256:4294967295] 192.168.202.110@tcp LPNI seq info [330880:330880:8:4294967295]
00000400:00000200:3.0:1788575488.407069:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575488.407075:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575488.407079:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575488.407082:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575488.407085:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575488.407088:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575488.407092:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575488.407095:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575488.407098:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575488.407100:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575488.407103:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575488.407107:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575488.407109:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575488.407111:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575488.407113:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575488.407117:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.407124:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575488.407127:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575513 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.407132:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.407134:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575488.407137:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575488.407140:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575488.407143:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575488.407146:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307092153
00000100:00000001:3.0:1788575488.407148:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307092922
00000100:00000001:3.0:1788575488.407150:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307092153
00000100:00000001:3.0:1788575488.407152:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307092922
00000100:00000001:3.0:1788575488.407154:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575488.407157:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.407158:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575488.407161:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575488/real 1788575488]  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575488.407168:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575488.407170:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575488.407171:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575488.407173:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575488.407175:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575488.407177:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575488.407180:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575488.407187:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.407198:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575488.407200:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.407202:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575488.407203:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575488.407206:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.407212:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575488.407216:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.407222:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575488.407224:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.407226:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575488.407227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.407229:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575488.407230:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575488.407232:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575488.407235:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575488.407237:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575488.407241:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575488.407248:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213440:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575488.407251:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.407253:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575488.407256:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393213312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.407261:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575488.407263:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575488.407265:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575488.407267:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575488.407269:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575488.407270:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575488.407272:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575488.407274:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575488.407276:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575488.407277:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575488.407279:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575488.407281:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575488.407282:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.407284:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575488.407287:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393213440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575513 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575488.407293:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575488.407294:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575488.407296:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575488.407298:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575488.407299:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575488.407300:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575488.407302:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575488.407303:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575488.407304:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575488.407305:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575488.407306:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575488.407308:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575488.407309:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.407310:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575488.407313:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575488.407314:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575488.407316:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575488.407317:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575488.789557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575488.789564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575488.789569:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575488.789570:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575488.789572:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575488.789574:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575488.789639:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575488.789642:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575488.789645:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575488.917584:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575488.917592:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575488.917594:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575488.917599:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575488.917603:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575488.917606:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575488.917608:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575488.917610:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575488.917613:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575489.365553:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307092922, at 4307092928
00000100:00000040:1.0:1788575489.365553:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307092922, at 4307092928
00000100:00000010:0.0:1788575489.365573:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:1.0:1788575489.365576:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00010000:00000001:3.0:1788575489.813526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575489.813531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575489.813536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575489.813537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575489.813540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575489.813542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575489.813606:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575489.813611:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575489.813614:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575489.941531:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575489.941538:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575489.941540:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575489.941545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575489.941549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575489.941553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575489.941556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575489.941559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575489.941563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575490.837510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575490.837538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575490.837542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575490.837543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575490.837545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575490.837546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575490.837590:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575490.837592:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575490.837595:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575490.965551:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575490.965557:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575490.965559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575490.965564:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575490.965568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575490.965572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575490.965575:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575490.965577:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575490.965580:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575491.861515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575491.861521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575491.861524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575491.861525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575491.861527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575491.861528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575491.861592:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575491.861595:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575491.861598:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575491.989500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575491.989506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575491.989508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575491.989513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575491.989516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575491.989520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575491.989523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575491.989525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575491.989528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575492.885502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575492.885508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575492.885513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575492.885515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575492.885518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575492.885519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575492.885560:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575492.885564:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575492.885567:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575493.013515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575493.013522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575493.013524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575493.013529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575493.013533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575493.013537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575493.013540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575493.013543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575493.013546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575493.525482:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575493.525490:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575493.525492:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575493.525495:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575493.525498:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575493.525504:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12424
00000400:00000200:2.0:1788575493.525511:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575493.525517:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575493.525519:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575493.525522:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525531:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575493.525533:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525535:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575493.525538:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575493.525539:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575493.525541:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575493.525544:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575493.525545:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575493.525546:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575493.525548:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525549:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575493.525551:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525553:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575493.525556:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575493.525559:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525561:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575493.525563:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575493.525565:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575493.525567:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575493.525569:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575493.525571:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575493.525577:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03480.
00000020:00000040:2.0:1788575493.525580:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575493.525585:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575493.525587:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575493.525602:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575493.525604:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575493.525607:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575493.525613:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000100:00000001:2.0:1788575493.525618:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575493.525626:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575493.525637:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575493.525640:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575493.525644:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9656041] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575493.525659:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575493.525661:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575493.525663:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575493.525664:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575493.525665:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575493.525667:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575493.525669:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12424
00000400:00000200:2.0:1788575493.525671:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575493.525673:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575493.525674:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575493.525676:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525678:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575493.525679:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525681:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575493.525683:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575493.525684:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575493.525685:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575493.525687:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575493.525688:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575493.525689:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575493.525690:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575493.525694:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1788575493.525695:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525698:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575493.525699:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525700:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575493.525702:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575493.525702:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1788575493.525705:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525708:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575493.525710:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1788575493.525711:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575493.525715:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575493.525719:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575493.525723:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575493.525725:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:2.0:1788575493.525727:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575493.525728:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788575493.525733:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575493.525734:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1788575493.525736:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575493.525737:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:2.0:1788575493.525739:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01180.
02000000:00000001:3.0:1788575493.525740:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575493.525744:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393213568:192.168.202.110@tcp:250:kworker.0
00000020:00000040:2.0:1788575493.525749:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:3.0:1788575493.525751:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575493.525753:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575493.525754:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575493.525755:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1788575493.525756:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575493.525758:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575493.525759:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575493.525762:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1788575493.525763:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575493.525768:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
02000000:00000010:3.0:1788575493.525768:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575493.525771:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575493.525773:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575493.525778:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575493.525781:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00100000:2.0:1788575493.525783:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575493.525787:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575493.525791:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575493.525793:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575493.525795:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393213568, portal 25
00000100:00000040:2.0:1788575493.525796:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b74282d7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.525800:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575493.525802:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000001:2.0:1788575493.525809:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575493.525813:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575493.525813:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575493.525821:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575493.525823:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575493.525826:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575493.525829:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:3.0:1788575493.525829:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575493.525832:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575493.525834:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12424
00000400:00000010:3.0:1788575493.525836:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000400:00000200:2.0:1788575493.525839:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:3.0:1788575493.525841:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393213568, offset 0
00000400:00000200:2.0:1788575493.525844:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575493.525846:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575493.525850:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575493.525851:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525853:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575493.525855:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525856:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575493.525860:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1788575493.525860:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575493.525863:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575493.525865:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575493.525866:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575493.525869:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:2.0:1788575493.525870:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575493.525874:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575493.525875:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1788575493.525880:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575493.525882:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1788575493.525883:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575493.525887:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1788575493.525888:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575493.525892:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525893:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575493.525896:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575493.525897:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330924:330924:256:4294967295] 192.168.202.110@tcp LPNI seq info [330881:330881:8:4294967295]
00000400:00000200:3.0:1788575493.525901:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575493.525905:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575493.525907:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575493.525909:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575493.525913:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575493.525914:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1788575493.525915:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575493.525919:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575493.525921:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575493.525925:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575493.525927:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575493.525927:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000010:2.0:1788575493.525933:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000020:00000040:2.0:1788575493.525936:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000010:3.0:1788575493.525936:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
02000000:00000001:2.0:1788575493.525939:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575493.525941:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575493.525942:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575493.525943:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575493.525946:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575493.525947:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:2.0:1788575493.525952:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000800:00000200:3.0:1788575493.525954:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1788575493.525958:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575493.525963:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575493.525968:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575493.525973:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00100000:2.0:1788575493.525976:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:3.0:1788575493.525977:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575493.525987:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1788575493.525988:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575493.525992:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575493.525996:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1788575493.525999:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575493.526004:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000100:00000001:2.0:1788575493.526006:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575493.526007:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000040:2.0:1788575493.526009:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575493.526012:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575493.526018:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12434)
00000100:00000001:3.0:1788575493.526018:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575493.526024:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526032:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575493.526035:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575518 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526040:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575493.526042:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575493.526048:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575493.526052:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575493.526055:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307098041
00000100:00000010:3.0:1788575493.526060:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575493.526066:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575493.526079:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307098041
00000100:00000010:3.0:1788575493.526083:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788575493.526086:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575493.526089:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575493.526092:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575493.526094:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575493.526099:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575493/real 1788575493]  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575493.526109:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575493.526111:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575493.526113:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575493.526116:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575493.526118:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575493.526120:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575493.526124:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575493.526132:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526136:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575493.526137:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526139:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575493.526140:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575493.526143:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526147:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575493.526150:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526154:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575493.526156:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526156:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575493.526157:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526158:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575493.526160:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575493.526162:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575493.526164:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575493.526165:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575493.526167:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575493.526172:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393213568:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575493.526176:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526178:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575493.526180:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393213568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526184:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575493.526185:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575493.526187:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575493.526189:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575493.526190:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575493.526191:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575493.526193:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575493.526195:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575493.526198:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575493.526200:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575493.526202:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:3.0:1788575493.526204:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575493.526206:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526207:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526214:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575493.526216:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575493.526218:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575493.526220:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575493.526221:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575493.526225:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526231:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575493.526232:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575493.526233:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575493.526235:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575493.526236:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575493.526238:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213696:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575493.526240:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575493.526242:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575493.526242:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575493.526244:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575493.526246:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575493.526247:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575493.526250:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575493.526251:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575493.526254:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575493.526256:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393213696, portal 10
00000100:00000001:3.0:1788575493.526257:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575493.526258:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000040:3.0:1788575493.526262:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526266:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575493.526268:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575493.526270:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575493.526272:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393213696, offset 0
00000400:00000200:3.0:1788575493.526276:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575493.526280:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575493.526283:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575493.526286:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575493.526288:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575493.526291:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575493.526293:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575493.526296:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330925:330925:256:4294967295] 192.168.202.110@tcp LPNI seq info [330882:330882:8:4294967295]
00000400:00000200:3.0:1788575493.526300:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575493.526306:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575493.526310:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575493.526313:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575493.526315:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575493.526317:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788575493.526320:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575493.526322:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575493.526324:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575493.526325:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575493.526327:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575493.526329:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575493.526332:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575493.526334:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575493.526336:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575493.526340:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526347:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575493.526350:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575518 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526356:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575493.526358:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575493.526361:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575493.526365:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575493.526376:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575493.526378:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307097126
00000100:00000001:3.0:1788575493.526380:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307098041
00000100:00000001:3.0:1788575493.526381:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307097126
00000100:00000001:3.0:1788575493.526382:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307098041
00000100:00000001:3.0:1788575493.526383:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575493.526385:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575493.526386:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575493.526389:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575493/real 1788575493]  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575493.526408:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575493.526409:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575493.526410:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575493.526412:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575493.526413:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575493.526414:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575493.526417:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575493.526422:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526425:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575493.526426:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526427:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575493.526428:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575493.526430:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526434:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575493.526436:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526440:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575493.526441:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526442:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575493.526442:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526443:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575493.526445:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575493.526446:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575493.526448:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575493.526449:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575493.526451:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575493.526455:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213696:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575493.526457:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575493.526459:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526467:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575493.526468:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575493.526469:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575493.526470:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575493.526471:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575493.526473:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575493.526475:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575493.526476:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575493.526477:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575493.526478:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575493.526480:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575493.526481:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575493.526483:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575493.526485:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575493.526487:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575493.526488:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393213824, portal 4
00000100:00000001:3.0:1788575493.526490:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575493.526490:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575493.526494:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526498:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575493.526499:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575493.526500:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575493.526501:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393213824, offset 0
00000400:00000200:3.0:1788575493.526504:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575493.526506:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575493.526508:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575493.526510:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575493.526511:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575493.526513:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575493.526514:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575493.526517:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330926:330926:256:4294967295] 192.168.202.110@tcp LPNI seq info [330883:330883:8:4294967295]
00000400:00000200:3.0:1788575493.526519:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575493.526523:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575493.526526:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575493.526528:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575493.526530:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575493.526531:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788575493.526534:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575493.526535:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575493.526538:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575493.526539:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575493.526540:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575493.526542:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575493.526545:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575493.526546:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575493.526548:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575493.526551:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526558:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575493.526561:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575518 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526567:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575493.526568:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575493.526571:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575493.526574:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575493.526577:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575493.526579:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307097273
00000100:00000001:3.0:1788575493.526581:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307098041
00000100:00000001:3.0:1788575493.526583:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307097273
00000100:00000001:3.0:1788575493.526585:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307098041
00000100:00000001:3.0:1788575493.526587:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575493.526602:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575493.526605:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575493.526608:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575493/real 1788575493]  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575493.526613:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575493.526614:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575493.526615:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575493.526617:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575493.526618:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575493.526619:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575493.526621:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575493.526626:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526631:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575493.526632:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526634:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575493.526635:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575493.526638:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526644:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575493.526648:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526655:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575493.526657:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526658:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575493.526660:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526661:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575493.526663:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575493.526666:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575493.526669:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575493.526671:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575493.526674:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575493.526679:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393213824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575493.526681:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526683:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575493.526684:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393213696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526688:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575493.526689:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575493.526691:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575493.526692:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575493.526693:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575493.526694:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575493.526696:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575493.526697:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575493.526698:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575493.526699:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575493.526700:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:3.0:1788575493.526701:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575493.526702:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526703:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575493.526705:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393213824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575518 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575493.526711:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575493.526712:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575493.526714:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575493.526716:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575493.526717:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575493.526718:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575493.526719:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575493.526721:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575493.526723:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575493.526724:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575493.526725:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575493.526728:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575493.526729:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526730:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575493.526734:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575493.526735:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575493.526736:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575493.526738:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575493.909539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575493.909545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575493.909550:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575493.909551:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575493.909555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575493.909556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575493.909623:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575493.909626:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575493.909628:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575494.037492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575494.037498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575494.037500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575494.037504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575494.037507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575494.037509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575494.037511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575494.037513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575494.037514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575494.485483:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307098041, at 4307098048
00000100:00000040:0.0:1788575494.485486:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307098041, at 4307098048
00000100:00000010:0.0:1788575494.485498:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788575494.485501:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00010000:00000001:3.0:1788575494.933527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575494.933534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575494.933538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575494.933542:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575494.933544:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575494.933547:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575494.933550:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575494.933551:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575494.933553:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575494.933555:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575494.933557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575494.933558:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575494.933559:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575494.933560:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575494.933561:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575494.933562:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575494.933563:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575494.933564:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575494.933657:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575494.933661:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575494.933665:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575495.061520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575495.061529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575495.061531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575495.061538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575495.061542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575495.061547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575495.061550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575495.061553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575495.061557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575495.957533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575495.957541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575495.957548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575495.957550:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575495.957554:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575495.957556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575495.957630:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575495.957635:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575495.957639:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575496.085485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575496.085495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575496.085497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575496.085503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575496.085507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575496.085511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575496.085514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575496.085517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575496.085521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575496.981472:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575496.981478:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575496.981480:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575496.981482:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575496.981485:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575496.981486:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575496.981488:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575496.981489:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575496.981491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575496.981492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575496.981494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575496.981495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575496.981545:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575496.981548:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575496.981550:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575497.109503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575497.109509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575497.109511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575497.109515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575497.109517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575497.109520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575497.109522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575497.109524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575497.109525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575498.005547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575498.005559:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575498.005566:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575498.005567:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575498.005571:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575498.005572:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575498.005658:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575498.005669:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575498.005672:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575498.135853:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575498.135860:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575498.135861:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575498.135865:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575498.135868:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575498.135871:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575498.135874:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575498.135876:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575498.135879:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575498.645548:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575498.645564:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575498.645567:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575498.645572:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575498.645578:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575498.645587:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12429
00000400:00000200:0.0:1788575498.645598:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575498.645605:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575498.645609:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575498.645613:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575498.645620:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575498.645622:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575498.645624:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575498.645628:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575498.645631:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575498.645635:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575498.645638:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575498.645641:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575498.645644:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575498.645647:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575498.645649:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575498.645654:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575498.645656:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575498.645660:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575498.645665:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575498.645668:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575498.645672:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575498.645675:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575498.645677:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575498.645680:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575498.645682:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575498.645693:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788575498.645697:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575498.645702:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575498.645704:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575498.645707:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575498.645711:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575498.645716:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575498.645725:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788575498.645734:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575498.645754:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575498.645766:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575498.645771:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575498.645776:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575498.645805:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575498.645807:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575498.645810:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575498.645811:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575498.645813:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575498.645815:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575498.645817:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12429
00000400:00000200:0.0:1788575498.645821:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575498.645824:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575498.645826:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575498.645829:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575498.645830:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575498.645833:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575498.645833:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575498.645840:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575498.645841:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575498.645844:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1788575498.645846:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575498.645849:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575498.645853:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:3.0:1788575498.645855:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575498.645856:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575498.645862:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575498.645867:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575498.645871:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1788575498.645874:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575498.645875:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575498.645877:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575498.645880:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575498.645882:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575498.645882:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1788575498.645886:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575498.645886:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1788575498.645891:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575498.645892:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575498.645893:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575498.645897:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:3.0:1788575498.645899:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393213952:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788575498.645904:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575498.645905:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575498.645908:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575498.645910:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575498.645912:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:0.0:1788575498.645915:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575498.645918:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575498.645919:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575498.645925:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575498.645929:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:0.0:1788575498.645933:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575498.645937:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575498.645940:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575498.645942:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575498.645942:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
00000100:00000001:0.0:1788575498.645947:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575498.645947:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575498.645951:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575498.645953:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575498.645958:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1788575498.645961:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000400:00000010:3.0:1788575498.645964:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:0.0:1788575498.645967:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575498.645972:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575498.645979:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1788575498.645980:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:0.0:1788575498.645981:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575498.645987:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575498.645991:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393213952, portal 25
00000100:00000001:0.0:1788575498.645993:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575498.645996:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1788575498.645997:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575498.646000:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00080000:0.0:1788575498.646001:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575498.646004:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575498.646006:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575498.646008:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:3.0:1788575498.646011:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788575498.646012:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12429
00000400:00000200:0.0:1788575498.646019:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575498.646023:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575498.646028:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575498.646030:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575498.646032:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575498.646035:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575498.646036:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788575498.646039:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575498.646042:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:3.0:1788575498.646042:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000020:00000001:0.0:1788575498.646047:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575498.646048:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575498.646050:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:3.0:1788575498.646050:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393213952, offset 0
00000020:00000040:0.0:1788575498.646054:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575498.646058:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575498.646061:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788575498.646062:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575498.646068:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575498.646070:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575498.646074:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575498.646075:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575498.646078:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575498.646082:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575498.646087:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:0.0:1788575498.646089:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575498.646091:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575498.646093:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575498.646096:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575498.646099:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575498.646101:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575498.646102:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00100000:0.0:1788575498.646105:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:3.0:1788575498.646110:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00000001:0.0:1788575498.646111:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575498.646115:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575498.646117:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000010:0.0:1788575498.646120:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000400:00000200:3.0:1788575498.646121:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:0.0:1788575498.646126:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575498.646130:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330927:330927:256:4294967295] 192.168.202.110@tcp LPNI seq info [330884:330884:8:4294967295]
02000000:00000001:0.0:1788575498.646132:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575498.646136:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575498.646140:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1788575498.646140:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575498.646143:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575498.646145:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575498.646149:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788575498.646155:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575498.646157:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00100000:0.0:1788575498.646161:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.0:1788575498.646169:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575498.646173:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575498.646176:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575498.646177:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000100:00000040:0.0:1788575498.646182:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788575498.646190:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575498.646198:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575498.646199:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1788575498.646203:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.1:1788575498.646209:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788575498.646211:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:3.1:1788575498.646215:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1788575498.646219:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12439)
00000800:00000100:3.0:1788575498.646222:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575498.646226:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575498.646234:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575498.646241:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575498.646248:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575498.646250:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575498.646255:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575498.646259:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646270:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575498.646273:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575523 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646282:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575498.646284:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575498.646291:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575498.646296:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575498.646299:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307103161
00000100:00000010:3.0:1788575498.646305:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575498.646313:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575498.646328:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307103161
00000100:00000010:3.0:1788575498.646332:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575498.646336:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575498.646340:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575498.646342:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575498.646345:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575498.646351:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575498/real 1788575498]  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575498.646362:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575498.646365:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575498.646367:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575498.646370:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575498.646372:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575498.646375:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575498.646379:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575498.646416:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646424:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575498.646426:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.646428:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575498.646431:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575498.646435:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646442:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575498.646447:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646455:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575498.646458:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.646459:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575498.646460:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.646463:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575498.646465:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575498.646468:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575498.646472:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575498.646475:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575498.646479:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575498.646488:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393213952:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575498.646492:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.646494:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575498.646497:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393213952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646510:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575498.646513:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575498.646516:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575498.646520:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575498.646522:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575498.646524:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575498.646526:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575498.646530:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:3.0:1788575498.646534:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575498.646535:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575498.646537:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:3.0:1788575498.646541:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575498.646542:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.646544:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.646553:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575498.646555:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575498.646557:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575498.646558:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575498.646560:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575498.646564:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646571:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575498.646572:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575498.646581:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575498.646583:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575498.646586:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575498.646589:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575498.646592:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575498.646594:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575498.646596:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575498.646598:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575498.646601:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575498.646603:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575498.646606:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575498.646610:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575498.646614:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575498.646617:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393214080, portal 10
00000100:00000001:3.0:1788575498.646619:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575498.646620:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00000040:3.0:1788575498.646627:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646634:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575498.646636:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575498.646639:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575498.646641:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393214080, offset 0
00000400:00000200:3.0:1788575498.646645:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575498.646648:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575498.646651:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575498.646654:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575498.646657:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575498.646659:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575498.646662:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575498.646665:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330928:330928:256:4294967295] 192.168.202.110@tcp LPNI seq info [330885:330885:8:4294967295]
00000400:00000200:3.0:1788575498.646669:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575498.646676:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575498.646680:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575498.646683:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575498.646687:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575498.646690:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788575498.646695:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575498.646698:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788575498.646701:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575498.646703:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575498.646706:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575498.646710:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575498.646713:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575498.646715:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575498.646718:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575498.646721:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646728:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575498.646731:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575523 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646751:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575498.646753:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575498.646756:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575498.646760:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575498.646764:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575498.646766:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307102247
00000100:00000001:3.0:1788575498.646769:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307103161
00000100:00000001:3.0:1788575498.646771:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307102247
00000100:00000001:3.0:1788575498.646773:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307103161
00000100:00000001:3.0:1788575498.646775:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575498.646777:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575498.646779:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575498.646783:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575498/real 1788575498]  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575498.646791:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575498.646793:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575498.646795:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575498.646797:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575498.646799:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575498.646801:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575498.646804:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575498.646812:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646824:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575498.646825:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.646827:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575498.646829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575498.646833:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646840:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575498.646844:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646850:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575498.646852:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.646854:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575498.646855:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.646857:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575498.646859:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575498.646862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575498.646865:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575498.646868:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575498.646872:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575498.646880:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575498.646883:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575498.646886:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646893:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575498.646894:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575498.646897:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575498.646898:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575498.646901:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575498.646903:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214208:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575498.646907:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575498.646909:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575498.646910:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575498.646912:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575498.646915:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575498.646917:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575498.646921:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575498.646923:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575498.646927:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575498.646930:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393214208, portal 4
00000100:00000001:3.0:1788575498.646932:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575498.646933:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:3.0:1788575498.646940:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.646947:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575498.646949:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575498.646951:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575498.646954:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393214208, offset 0
00000400:00000200:3.0:1788575498.646958:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575498.646961:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575498.646964:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575498.646967:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575498.646970:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575498.646972:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575498.646974:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575498.646978:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330929:330929:256:4294967295] 192.168.202.110@tcp LPNI seq info [330886:330886:8:4294967295]
00000400:00000200:3.0:1788575498.646982:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575498.646989:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575498.646994:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575498.646997:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575498.647001:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575498.647004:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788575498.647008:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575498.647011:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788575498.647015:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575498.647017:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575498.647020:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575498.647024:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575498.647027:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575498.647029:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575498.647031:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575498.647034:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.647042:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575498.647045:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575523 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.647052:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575498.647054:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575498.647056:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575498.647059:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575498.647062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575498.647064:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307102393
00000100:00000001:3.0:1788575498.647067:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307103161
00000100:00000001:3.0:1788575498.647069:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307102393
00000100:00000001:3.0:1788575498.647071:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307103161
00000100:00000001:3.0:1788575498.647073:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575498.647075:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575498.647077:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575498.647081:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575498/real 1788575498]  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575498.647090:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575498.647092:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575498.647094:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575498.647096:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575498.647098:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575498.647100:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575498.647103:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575498.647112:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.647118:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575498.647119:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.647121:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575498.647122:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575498.647126:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.647132:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575498.647136:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.647143:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575498.647146:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.647147:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575498.647149:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.647150:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575498.647152:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575498.647155:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575498.647157:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575498.647159:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575498.647164:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575498.647173:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214208:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575498.647176:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.647178:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575498.647181:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393214080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.647187:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575498.647188:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575498.647190:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575498.647192:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575498.647194:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575498.647196:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575498.647198:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575498.647200:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:3.0:1788575498.647202:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575498.647204:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575498.647205:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:3.0:1788575498.647207:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575498.647209:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.647210:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575498.647214:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393214208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575523 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575498.647220:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575498.647222:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575498.647223:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575498.647226:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575498.647227:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575498.647228:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575498.647229:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575498.647231:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5c00.
02000000:00000001:3.0:1788575498.647233:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575498.647234:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575498.647236:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:3.0:1788575498.647238:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575498.647239:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.647241:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575498.647244:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575498.647245:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575498.647247:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575498.647248:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575499.029439:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575499.029445:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575499.029451:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575499.029452:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575499.029456:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575499.029457:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575499.029502:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575499.029505:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575499.029508:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575499.157529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575499.157535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575499.157537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575499.157540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575499.157543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575499.157546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575499.157548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575499.157549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575499.157552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575499.605534:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307103161, at 4307103168
00000100:00000040:1.0:1788575499.605534:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307103161, at 4307103168
00000100:00000010:1.0:1788575499.605553:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575499.605556:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575500.053507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575500.053514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575500.053519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575500.053520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575500.053523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575500.053524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575500.053590:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575500.053607:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575500.053613:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575500.181505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575500.181511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575500.181514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575500.181518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575500.181522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575500.181526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575500.181528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575500.181531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575500.181534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575501.077525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575501.077532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575501.077537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575501.077538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575501.077541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575501.077542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575501.077654:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575501.077657:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575501.077660:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575501.205521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575501.205527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575501.205529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575501.205533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575501.205536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575501.205538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575501.205540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575501.205542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575501.205544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575502.101507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575502.101519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575502.101524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575502.101525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575502.101529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575502.101530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575502.101595:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575502.101599:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575502.101602:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575502.229461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575502.229468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575502.229470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575502.229475:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575502.229479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575502.229482:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575502.229485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575502.229487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575502.229489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575503.125531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575503.125538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575503.125543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575503.125544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575503.125547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575503.125548:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575503.125610:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575503.125614:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575503.125617:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575503.253550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575503.253558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575503.253559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575503.253564:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575503.253567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575503.253571:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575503.253573:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575503.253575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575503.253578:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575503.765543:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575503.765556:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575503.765560:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575503.765565:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575503.765571:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575503.765580:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12434
00000400:00000200:3.0:1788575503.765592:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575503.765599:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575503.765603:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575503.765607:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575503.765611:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575503.765614:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575503.765616:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575503.765621:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575503.765623:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575503.765626:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575503.765630:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575503.765632:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575503.765635:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575503.765638:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575503.765640:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575503.765642:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575503.765644:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575503.765648:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575503.765651:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575503.765654:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575503.765658:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575503.765662:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575503.765665:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575503.765667:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575503.765671:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575503.765682:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575503.765687:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575503.765691:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575503.765693:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575503.765697:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575503.765701:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575503.765705:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575503.765715:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000100:00000001:3.0:1788575503.765724:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575503.765736:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575503.765747:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575503.765752:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575503.765757:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575503.765782:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575503.765786:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575503.765789:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575503.765791:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575503.765792:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575503.765796:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575503.765798:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12434
00000400:00000200:3.0:1788575503.765802:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575503.765806:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575503.765809:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575503.765812:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575503.765815:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575503.765817:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575503.765819:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575503.765820:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575503.765824:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575503.765829:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575503.765833:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575503.765837:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000040:2.0:1788575503.765839:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1788575503.765839:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575503.765843:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575503.765847:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575503.765852:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575503.765859:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575503.765860:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575503.765863:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575503.765865:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575503.765867:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575503.765871:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575503.765871:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575503.765876:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575503.765880:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575503.765894:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00100000:2.0:1788575503.765899:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393214336:192.168.202.110@tcp:250:kworker.0
00000100:00080000:3.0:1788575503.765901:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575503.765907:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.765908:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575503.765910:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575503.765916:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575503.765920:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575503.765921:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1788575503.765923:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575503.765926:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575503.765930:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575503.765931:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:3.0:1788575503.765936:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575503.765939:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575503.765940:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575503.765944:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575503.765944:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788575503.765949:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:2.0:1788575503.765955:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575503.765961:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
02000000:00000001:3.0:1788575503.765968:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1788575503.765970:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
02000000:00000001:3.0:1788575503.765970:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575503.765975:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575503.765976:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575503.765978:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575503.765980:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393214336, portal 25
00000100:00000001:2.0:1788575503.765984:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000010:3.0:1788575503.765984:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8c00.
00000100:00000001:2.0:1788575503.765988:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000001:3.0:1788575503.765990:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575503.765998:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575503.765999:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575503.766025:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575503.766026:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575503.766030:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575503.766034:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575503.766038:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575503.766039:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575503.766046:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393214336, offset 0
00000100:00000001:3.0:1788575503.766054:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575503.766058:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575503.766059:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575503.766065:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575503.766067:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575503.766068:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575503.766073:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575503.766073:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575503.766079:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12434
00000400:00000200:2.0:1788575503.766084:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575503.766091:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575503.766096:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575503.766100:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575503.766103:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575503.766108:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330930:330930:256:4294967295] 192.168.202.110@tcp LPNI seq info [330887:330887:8:4294967295]
00000400:00000200:2.0:1788575503.766112:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575503.766118:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575503.766124:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575503.766129:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1788575503.766129:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575503.766136:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575503.766140:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1788575503.766141:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575503.766144:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575503.766148:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000010:2.0:1788575503.766150:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000020:00000001:3.0:1788575503.766151:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575503.766154:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575503.766157:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575503.766160:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:2.0:1788575503.766164:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788575503.766164:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575503.766169:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.0:1788575503.766172:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000040:3.0:1788575503.766174:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575503.766179:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788575503.766184:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575503.766185:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575503.766190:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:3.0:1788575503.766191:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575503.766195:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000100:2.0:1788575503.766200:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575503.766201:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000010:2.0:1788575503.766206:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000100:00000001:3.0:1788575503.766206:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575503.766211:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575503.766214:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575503.766216:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:3.0:1788575503.766217:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575503.766222:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575503.766226:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575503.766227:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000008:00000001:3.0:1788575503.766231:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575503.766234:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575503.766236:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000100:00000010:3.0:1788575503.766239:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000400:00000010:2.0:1788575503.766241:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000020:00000040:3.0:1788575503.766244:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:2.0:1788575503.766248:0:2361:0:(events.c:35:request_out_callback()) Process entered
02000000:00000001:3.0:1788575503.766249:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575503.766252:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575503.766256:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575503.766256:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575503.766259:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575503.766263:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575503.766272:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:2.0:1788575503.766273:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1788575503.766278:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575503.766279:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575528 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575503.766287:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575503.766301:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575503.766303:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575503.766305:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.766307:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:3.0:1788575503.766312:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766316:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575503.766324:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575503.766325:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.766329:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307108281
00000100:00000040:3.0:1788575503.766330:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575503.766336:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:2.0:1788575503.766337:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cae0.
00000100:00000040:3.0:1788575503.766343:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12445)
00000100:00000001:2.0:1788575503.766346:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575503.766354:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307108281
00000100:00000010:2.0:1788575503.766367:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c660.
00000100:00000001:2.0:1788575503.766374:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575503.766378:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575503.766381:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.766384:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575503.766389:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575503/real 1788575503]  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575503.766439:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575503.766441:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575503.766443:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575503.766446:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575503.766448:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575503.766450:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575503.766454:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575503.766469:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766477:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575503.766479:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.766481:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575503.766484:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575503.766488:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766495:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575503.766500:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766507:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575503.766510:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.766512:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575503.766513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.766516:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575503.766518:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575503.766521:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575503.766525:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575503.766527:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575503.766532:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575503.766539:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393214336:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575503.766544:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.766546:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575503.766549:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393214336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766561:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575503.766564:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575503.766567:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575503.766570:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575503.766572:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575503.766574:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575503.766576:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575503.766579:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:2.0:1788575503.766583:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575503.766585:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575503.766587:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:2.0:1788575503.766591:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575503.766592:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.766595:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.766602:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575503.766605:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.766607:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575503.766608:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575503.766610:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575503.766613:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766618:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575503.766620:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575503.766621:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575503.766623:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575503.766625:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575503.766628:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214464:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575503.766630:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575503.766632:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575503.766634:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575503.766635:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575503.766638:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575503.766640:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575503.766643:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575503.766646:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575503.766650:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575503.766652:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393214464, portal 10
00000100:00000001:2.0:1788575503.766654:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575503.766655:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:2.0:1788575503.766661:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766668:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575503.766670:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575503.766672:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575503.766674:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393214464, offset 0
00000400:00000200:2.0:1788575503.766678:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575503.766681:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575503.766685:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575503.766688:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575503.766690:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575503.766693:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575503.766695:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575503.766698:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330931:330931:256:4294967295] 192.168.202.110@tcp LPNI seq info [330888:330888:8:4294967295]
00000400:00000200:2.0:1788575503.766702:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575503.766708:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575503.766712:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575503.766715:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575503.766718:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575503.766720:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575503.766724:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575503.766727:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1788575503.766730:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575503.766732:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575503.766734:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575503.766737:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575503.766740:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575503.766742:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575503.766745:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575503.766748:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766755:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575503.766758:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575528 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766763:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.766765:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575503.766768:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575503.766771:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575503.766775:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575503.766776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307107367
00000100:00000001:2.0:1788575503.766779:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307108281
00000100:00000001:2.0:1788575503.766782:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307107367
00000100:00000001:2.0:1788575503.766784:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307108281
00000100:00000001:2.0:1788575503.766786:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575503.766788:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.766790:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575503.766793:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575503/real 1788575503]  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575503.766800:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575503.766802:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575503.766803:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575503.766805:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575503.766807:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575503.766809:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575503.766812:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575503.766821:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766832:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575503.766834:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.766835:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575503.766837:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575503.766840:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766847:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575503.766850:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766855:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575503.766857:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.766858:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575503.766859:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.766860:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575503.766861:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575503.766863:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575503.766864:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575503.766866:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575503.766868:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575503.766873:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214464:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575503.766875:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575503.766877:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766880:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575503.766881:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575503.766895:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575503.766897:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575503.766898:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575503.766901:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214592:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575503.766903:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575503.766904:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575503.766905:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575503.766906:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575503.766909:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575503.766910:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575503.766913:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575503.766915:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575503.766917:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575503.766919:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393214592, portal 4
00000100:00000001:2.0:1788575503.766921:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575503.766922:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:2.0:1788575503.766928:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.766935:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575503.766937:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575503.766940:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000200:2.0:1788575503.766942:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393214592, offset 0
00000400:00000200:2.0:1788575503.766946:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575503.766950:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575503.766953:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575503.766956:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575503.766959:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575503.766961:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575503.766963:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575503.766966:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330932:330932:256:4294967295] 192.168.202.110@tcp LPNI seq info [330889:330889:8:4294967295]
00000400:00000200:2.0:1788575503.766970:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575503.766976:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575503.766979:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575503.766982:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000800:00000200:2.0:1788575503.766984:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575503.766986:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575503.766989:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575503.766991:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1788575503.766993:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575503.766995:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:2.0:1788575503.766997:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575503.766999:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575503.767001:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:2.0:1788575503.767002:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:2.0:1788575503.767004:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575503.767006:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.767011:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575503.767013:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575528 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.767017:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.767018:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575503.767020:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575503.767022:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575503.767024:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575503.767026:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307107513
00000100:00000001:2.0:1788575503.767027:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307108281
00000100:00000001:2.0:1788575503.767029:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307107513
00000100:00000001:2.0:1788575503.767030:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307108281
00000100:00000001:2.0:1788575503.767031:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575503.767032:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.767033:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575503.767035:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575503/real 1788575503]  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575503.767041:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575503.767042:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575503.767043:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575503.767044:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575503.767045:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575503.767046:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575503.767048:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575503.767052:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.767056:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575503.767057:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.767058:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575503.767059:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575503.767061:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.767064:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575503.767066:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.767070:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575503.767071:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.767072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575503.767073:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.767074:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575503.767075:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575503.767076:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575503.767078:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575503.767079:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575503.767082:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575503.767087:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214592:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575503.767089:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.767090:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575503.767091:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393214464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.767095:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575503.767096:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575503.767097:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575503.767098:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575503.767099:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575503.767100:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575503.767101:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575503.767101:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:2.0:1788575503.767103:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575503.767104:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575503.767104:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788575503.767106:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575503.767107:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.767108:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575503.767109:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393214592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575528 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575503.767113:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575503.767114:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575503.767114:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575503.767116:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575503.767116:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575503.767117:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575503.767118:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575503.767118:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575503.767120:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575503.767121:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575503.767122:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:2.0:1788575503.767124:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575503.767125:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.767126:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575503.767129:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575503.767130:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575503.767131:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575503.767132:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575504.149465:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575504.149471:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575504.149475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575504.149477:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575504.149479:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575504.149480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575504.149542:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575504.149547:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575504.149551:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575504.277530:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575504.277537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575504.277539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575504.277545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575504.277548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575504.277553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575504.277555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575504.277557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575504.277560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575504.725536:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307108281, at 4307108288
00000100:00000040:1.0:1788575504.725536:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307108281, at 4307108288
00000100:00000010:0.0:1788575504.725555:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000100:00000010:1.0:1788575504.725557:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00010000:00000001:3.0:1788575505.173555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575505.173562:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575505.173566:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575505.173569:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575505.173571:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575505.173574:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575505.173577:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575505.173579:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575505.173580:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575505.173582:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575505.173585:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575505.173585:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575505.173586:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575505.173587:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575505.173588:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575505.173588:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575505.173590:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575505.173591:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575505.173659:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575505.173663:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575505.173667:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575505.301489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575505.301498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575505.301499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575505.301504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575505.301508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575505.301512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575505.301514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575505.301516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575505.301519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575506.197506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575506.197513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575506.197518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575506.197520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575506.197523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575506.197525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575506.197604:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575506.197610:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575506.197613:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575506.325538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575506.325548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575506.325550:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575506.325557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575506.325561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575506.325565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575506.325567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575506.325569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575506.325572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575507.221600:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575507.221612:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575507.221615:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575507.221619:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575507.221626:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575507.221628:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575507.221630:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575507.221633:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575507.221637:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575507.221639:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575507.221643:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575507.221644:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575507.221695:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575507.221699:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575507.221702:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575507.349483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575507.349490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575507.349493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575507.349498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575507.349501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575507.349505:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575507.349507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575507.349509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575507.349512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575508.245512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575508.245521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575508.245529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575508.245530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575508.245534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575508.245536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575508.245604:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575508.245608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575508.245612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575508.373521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575508.373530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575508.373532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575508.373539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575508.373542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575508.373546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575508.373548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575508.373550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575508.373553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575508.885519:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575508.885530:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575508.885533:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575508.885537:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575508.885541:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575508.885548:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12440
00000400:00000200:3.0:1788575508.885558:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575508.885565:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575508.885567:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575508.885570:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885573:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575508.885575:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885576:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575508.885580:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575508.885581:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575508.885583:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575508.885586:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575508.885588:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575508.885590:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575508.885592:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885593:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575508.885595:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885596:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575508.885598:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575508.885600:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885602:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575508.885604:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575508.885608:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575508.885610:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575508.885612:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575508.885615:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575508.885625:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000020:00000040:3.0:1788575508.885628:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575508.885632:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575508.885633:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575508.885636:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575508.885639:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575508.885643:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575508.885649:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575508.885656:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575508.885664:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575508.885673:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575508.885676:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575508.885680:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575508.885695:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575508.885697:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575508.885700:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575508.885701:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575508.885702:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575508.885704:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575508.885706:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12440
00000400:00000200:3.0:1788575508.885708:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575508.885710:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575508.885712:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575508.885714:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885716:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575508.885718:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885719:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575508.885721:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575508.885722:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1788575508.885722:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575508.885724:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575508.885727:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575508.885729:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575508.885730:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575508.885730:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575508.885735:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575508.885736:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575508.885736:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575508.885739:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885741:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575508.885742:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575508.885746:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:2.0:1788575508.885748:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575508.885748:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575508.885751:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575508.885757:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575508.885760:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575508.885763:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575508.885764:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575508.885768:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575508.885768:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575508.885773:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575508.885775:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575508.885776:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:3.0:1788575508.885778:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1180.
02000000:00000001:2.0:1788575508.885780:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1788575508.885782:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575508.885785:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575508.885785:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575508.885788:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575508.885790:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575508.885792:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:2.0:1788575508.885793:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393214720:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575508.885794:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575508.885800:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:2.0:1788575508.885801:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575508.885803:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575508.885807:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575508.885810:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575508.885810:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575508.885814:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575508.885820:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575508.885823:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575508.885823:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575508.885829:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575508.885832:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575508.885836:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000040:3.0:1788575508.885837:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca1dca9a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575508.885844:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000001:3.0:1788575508.885851:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575508.885854:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393214720, portal 25
00000100:00080000:3.0:1788575508.885855:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575508.885859:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1788575508.885861:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575508.885862:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000001:3.0:1788575508.885865:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575508.885868:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1788575508.885871:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575508.885871:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575508.885877:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12440
00000400:00000200:3.0:1788575508.885883:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575508.885884:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:3.0:1788575508.885889:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:2.0:1788575508.885893:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575508.885893:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000010:2.0:1788575508.885896:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65cc0.
00000100:00000040:3.0:1788575508.885899:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575508.885902:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393214720, offset 0
00000100:00000001:3.0:1788575508.885902:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575508.885906:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885908:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575508.885911:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1788575508.885912:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575508.885913:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575508.885915:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575508.885917:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575508.885918:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575508.885920:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575508.885922:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885923:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575508.885924:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885925:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575508.885928:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575508.885929:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575508.885932:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575508.885936:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575508.885938:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575508.885941:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575508.885942:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:3.0:1788575508.885948:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575508.885951:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788575508.885953:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575508.885957:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575508.885959:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00000001:3.0:1788575508.885962:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575508.885967:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575508.885968:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575508.885973:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000010:3.0:1788575508.885976:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000400:00000200:2.0:1788575508.885980:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330933:330933:256:4294967295] 192.168.202.110@tcp LPNI seq info [330890:330890:8:4294967295]
00000020:00000040:3.0:1788575508.885981:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575508.885987:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575508.885990:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575508.885991:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575508.885994:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575508.885996:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575508.885998:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575508.886001:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575508.886005:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575508.886010:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00100000:3.0:1788575508.886012:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1788575508.886021:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575508.886022:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575508.886026:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575508.886044:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575508.886050:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575508.886051:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133f00.
00000100:00000040:3.0:1788575508.886052:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575508.886056:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575508.886060:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12450)
00000800:00000200:2.0:1788575508.886061:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575508.886067:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575508.886082:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575508.886085:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575508.886089:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307113401
00000100:00000010:2.0:1788575508.886093:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ccc0.
00000100:00000001:2.0:1788575508.886099:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575508.886108:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307113401
00000100:00000010:2.0:1788575508.886110:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c4e0.
00000100:00000001:2.0:1788575508.886113:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575508.886116:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575508.886118:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575508.886121:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575508.886124:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575508.886130:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575508.886132:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575508.886133:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575508.886135:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575508.886136:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575508.886139:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575508.886145:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575508.886146:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575508.886148:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575508.886149:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575508.886150:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575508.886152:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575508.886155:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575508.886156:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575508.886157:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575508.886158:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575508.886161:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575508.886163:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575508.886165:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575508.886168:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575508.886172:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575508.886174:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393214848, portal 10
00000100:00000001:2.0:1788575508.886176:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575508.886176:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000100:00000040:2.0:1788575508.886181:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575508.886186:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575508.886188:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575508.886190:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65dd0.
00000100:00000200:2.0:1788575508.886191:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393214848, offset 0
00000400:00000200:2.0:1788575508.886194:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575508.886196:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575508.886199:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575508.886201:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575508.886203:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575508.886204:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575508.886206:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575508.886208:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330934:330934:255:4294967295] 192.168.202.110@tcp LPNI seq info [330891:330891:7:4294967295]
00000400:00000200:2.0:1788575508.886210:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575508.886214:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575508.886217:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575508.886220:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133a00.
00000800:00000200:2.0:1788575508.886223:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575508.886226:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575508.886227:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575508.886230:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575508.886232:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575508.886233:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307112486
00000100:00000001:2.0:1788575508.886235:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307113401
00000100:00000001:2.0:1788575508.886237:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307112486
00000100:00000001:2.0:1788575508.886238:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307113401
00000100:00000001:2.0:1788575508.886240:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575508.886242:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575508.886243:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575508.886245:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575508.886250:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575508.886251:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575508.886252:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575508.886253:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575508.886255:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575508.886257:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575508.886258:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575508.886259:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575508.886260:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575508.886261:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575508.886263:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575508.886264:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575508.886266:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575508.886268:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575508.886270:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575508.886272:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393214976, portal 4
00000100:00000001:2.0:1788575508.886273:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575508.886274:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:2.0:1788575508.886279:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575508.886283:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575508.886284:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575508.886286:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65ee0.
00000100:00000200:2.0:1788575508.886287:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393214976, offset 0
00000400:00000200:2.0:1788575508.886290:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575508.886293:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575508.886295:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575508.886297:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575508.886299:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575508.886300:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575508.886302:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575508.886303:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330935:330935:254:4294967295] 192.168.202.110@tcp LPNI seq info [330892:330892:6:4294967295]
00000400:00000200:2.0:1788575508.886306:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575508.886309:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575508.886311:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575508.886313:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133900.
00000800:00000200:2.0:1788575508.886315:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575508.886317:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575508.886318:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575508.886320:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575508.886322:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575508.886323:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307112632
00000100:00000001:2.0:1788575508.886325:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307113401
00000100:00000001:2.0:1788575508.886326:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307112632
00000100:00000001:2.0:1788575508.886328:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307113401
00000100:00000001:2.0:1788575508.886329:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575508.886330:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575508.886331:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575508.886333:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575508.886334:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575508.886335:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575508.886336:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575508.886337:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575508.886337:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575508.886339:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575509.269542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575509.269552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575509.269560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575509.269561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575509.269563:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575509.269564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575509.269616:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575509.269620:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575509.269623:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575509.397446:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575509.397452:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575509.397453:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575509.397457:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575509.397459:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575509.397462:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575509.397464:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575509.397466:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575509.397468:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575509.845491:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307113401, at 4307113408
00000100:00000040:1.0:1788575509.845491:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307113401, at 4307113408
00000100:00000010:0.0:1788575509.845505:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000100:00000010:1.0:1788575509.845508:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00010000:00000001:3.0:1788575510.293496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575510.293501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575510.293505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575510.293506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575510.293508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575510.293509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575510.293565:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575510.293568:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575510.293571:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575510.421496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575510.421501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575510.421502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575510.421507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575510.421509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575510.421512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575510.421515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575510.421517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575510.421520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575511.317497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575511.317502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575511.317506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575511.317506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575511.317509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575511.317509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575511.317559:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575511.317563:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575511.317566:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575511.445531:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575511.445537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575511.445538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575511.445542:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575511.445545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575511.445549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575511.445552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575511.445554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575511.445556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788575511.957735:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788575511.957781:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788575511.957790:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575511.957794:0:2350:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788575511.957798:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575511.957802:0:2350:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788575511.957807:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575511.957811:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133f00.
00000400:00000200:1.0:1788575511.957816:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575511.957822:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575511.957829:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65cc0
00000400:00000010:1.0:1788575511.957832:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65cc0.
00000100:00000001:1.0:1788575511.957837:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575511.957847:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575511.957883:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575511.957888:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575511.957895:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575511.957897:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575511.957910:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575511.957912:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133a00.
00000400:00000200:1.0:1788575511.957915:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575511.957918:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575511.957921:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65dd0
00000400:00000010:1.0:1788575511.957922:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65dd0.
00000100:00000001:1.0:1788575511.957925:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575511.957937:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575511.957948:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575511.957951:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575511.957956:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575511.957959:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575511.957963:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575511.957965:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133900.
00000400:00000200:1.0:1788575511.957968:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575511.957970:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575511.957973:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65ee0
00000400:00000010:1.0:1788575511.957975:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65ee0.
00000100:00000001:1.0:1788575511.957977:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575511.957981:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575511.957987:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575511.957989:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575511.957995:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575511.957997:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575511.958075:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575511.958082:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575511.958085:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575511.958093:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575508/real 1788575511]  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575511.958104:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575511.958106:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575511.958110:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575511.958113:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575511.958114:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575511.958144:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575511.958148:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575511.958156:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.958163:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575511.958166:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.958169:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575511.958172:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575511.958175:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.958181:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575511.958186:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.958194:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575511.958197:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.958198:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575511.958199:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.958202:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575511.958206:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575511.958209:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575511.958220:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575511.958222:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575511.958226:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575511.958234:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575511.958238:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575511.958242:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575508/real 1788575511]  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575511.958248:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575511.958249:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575511.958251:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575511.958253:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575511.958254:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575511.958256:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575511.958258:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575511.958268:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.958273:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575511.958274:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.958276:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575511.958277:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575511.958280:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.958286:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575511.958289:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.958294:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575511.958297:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.958298:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575511.958299:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.958300:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575511.958303:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575511.958310:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575511.958315:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575511.958317:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575511.958320:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575511.958327:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393214976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575511.958332:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575511.958335:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575508/real 1788575511]  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575511.958344:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575511.958345:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575511.958346:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575511.958349:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575511.958351:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575511.958353:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575511.958356:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575511.958364:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.958370:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575511.958371:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.958373:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575511.958374:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575511.958377:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.958383:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575511.958386:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.959094:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575511.959097:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.959099:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575511.959100:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.959102:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575511.959104:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575511.959107:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575511.959112:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575511.959115:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575511.959130:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575511.959138:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393214720:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575511.959142:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.959145:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575511.959148:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393214848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.959154:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575511.959157:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575511.959162:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575511.959166:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575511.959169:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575511.959171:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575511.959174:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575511.959177:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575511.959181:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575511.959183:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575511.959186:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:2.0:1788575511.959189:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575511.959190:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.959192:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575511.959195:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393214976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.959201:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575511.959202:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575511.959203:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575511.959205:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575511.959206:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575511.959207:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575511.959209:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575511.959210:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575511.959213:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575511.959213:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575511.959214:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:2.0:1788575511.959218:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575511.959219:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.959221:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575511.959223:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393214720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575511.959230:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575511.959231:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575511.959233:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575511.959236:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575511.959237:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575511.959238:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575511.959240:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575511.959241:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575511.959243:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575511.959245:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575511.959246:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:2.0:1788575511.959248:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575511.959249:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.959253:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575511.959267:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575511.959270:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575511.959272:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575511.959274:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575512.341511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575512.341516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575512.341520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575512.341521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575512.341523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575512.341524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575512.341577:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575512.341581:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575512.341584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575512.469488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575512.469493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575512.469495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575512.469498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575512.469501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575512.469503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575512.469505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575512.469507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575512.469509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575513.365509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575513.365515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575513.365520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575513.365521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575513.365525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575513.365526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575513.365573:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575513.365576:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575513.365578:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575513.493486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575513.493492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575513.493494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575513.493498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575513.493500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575513.493503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575513.493505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575513.493507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575513.493509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575514.005480:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575514.005490:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575514.005493:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575514.005496:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575514.005501:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575514.005508:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12445
00000400:00000200:3.0:1788575514.005517:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575514.005525:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575514.005529:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575514.005533:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005536:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575514.005539:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005542:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575514.005546:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575514.005548:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575514.005551:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575514.005554:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575514.005556:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575514.005559:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575514.005562:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005564:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575514.005565:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005567:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575514.005571:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575514.005574:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005576:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575514.005578:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575514.005581:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575514.005584:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575514.005586:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575514.005589:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575514.005595:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788575514.005599:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575514.005603:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575514.005605:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575514.005608:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575514.005611:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575514.005615:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575514.005622:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575514.005628:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575514.005639:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575514.005649:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575514.005653:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575514.005658:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575514.005685:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575514.005689:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575514.005693:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575514.005694:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575514.005696:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575514.005699:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575514.005702:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12445
00000400:00000200:3.0:1788575514.005706:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575514.005710:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575514.005712:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575514.005716:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005719:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575514.005721:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005724:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575514.005727:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575514.005729:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575514.005731:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575514.005731:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575514.005734:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575514.005736:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575514.005737:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575514.005739:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575514.005742:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575514.005749:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575514.005749:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005754:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575514.005758:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005763:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575514.005765:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:3.0:1788575514.005767:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575514.005768:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575514.005771:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575514.005774:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575514.005775:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575514.005778:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575514.005779:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575514.005784:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575514.005787:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1788575514.005791:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393215104:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575514.005792:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575514.005796:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575514.005799:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:3.0:1788575514.005802:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575514.005805:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:3.0:1788575514.005807:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575514.005809:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575514.005810:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575514.005814:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:3.0:1788575514.005816:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
02000000:00000010:2.0:1788575514.005820:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
00000020:00000040:3.0:1788575514.005820:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575514.005824:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575514.005825:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575514.005828:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575514.005831:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575514.005834:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575514.005834:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575514.005838:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575514.005841:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
02000000:00000010:3.0:1788575514.005844:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575514.005850:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575514.005851:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00100000:3.0:1788575514.005857:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788575514.005859:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393215104, portal 25
00000100:00000001:2.0:1788575514.005865:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575514.005868:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000001:3.0:1788575514.005871:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575514.005874:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575514.005878:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575514.005882:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.005891:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575514.005897:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575514.005902:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000001:3.0:1788575514.005902:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575514.005908:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393215104, offset 0
00000100:00080000:3.0:1788575514.005908:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575514.005915:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575514.005917:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1788575514.005920:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575514.005920:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575514.005930:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575514.005931:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575514.005937:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12445
00000400:00000200:2.0:1788575514.005940:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575514.005946:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575514.005950:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575514.005954:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575514.005956:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575514.005960:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330936:330936:256:4294967295] 192.168.202.110@tcp LPNI seq info [330893:330893:8:4294967295]
00000400:00000200:2.0:1788575514.005964:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575514.005969:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575514.005977:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575514.005979:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1788575514.005981:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575514.005988:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.0:1788575514.005992:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575514.005993:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575514.005997:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575514.006000:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000100:00000001:3.0:1788575514.006000:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575514.006005:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575514.006006:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575514.006008:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.0:1788575514.006012:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1788575514.006012:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575514.006017:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.0:1788575514.006018:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:3.0:1788575514.006021:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575514.006028:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575514.006028:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575514.006034:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788575514.006035:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:3.0:1788575514.006037:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575514.006043:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000100:2.0:1788575514.006047:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575514.006048:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000010:2.0:1788575514.006052:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000100:00080000:3.0:1788575514.006054:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575514.006058:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575514.006060:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:3.0:1788575514.006061:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575514.006065:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575514.006070:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575514.006071:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1788575514.006075:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575514.006078:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575514.006080:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000008:00000001:3.0:1788575514.006081:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575514.006085:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:3.0:1788575514.006085:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575514.006091:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788575514.006093:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000020:00000040:3.0:1788575514.006097:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000200:2.0:1788575514.006100:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575514.006102:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575514.006105:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575514.006109:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575514.006112:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575514.006115:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575514.006121:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:2.0:1788575514.006126:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1788575514.006127:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575514.006132:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575539 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575514.006137:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575514.006147:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575514.006151:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575514.006154:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575514.006158:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575514.006159:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575514.006168:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575514.006169:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006173:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307118521
00000100:00000010:2.0:1788575514.006196:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c600.
00000100:00000001:2.0:1788575514.006202:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575514.006210:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575514.006211:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307118521
00000100:00000040:3.0:1788575514.006215:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:2.0:1788575514.006217:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cf00.
00000100:00000001:2.0:1788575514.006222:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1788575514.006223:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575514.006229:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1788575514.006231:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12455)
00000100:00000001:2.0:1788575514.006233:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575514.006237:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575514.006243:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575514/real 1788575514]  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575514.006252:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575514.006253:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575514.006256:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575514.006259:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575514.006261:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575514.006264:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575514.006267:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575514.006274:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006280:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575514.006282:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006285:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575514.006287:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575514.006291:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006297:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575514.006301:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006307:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575514.006309:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006311:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575514.006312:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006315:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575514.006317:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575514.006319:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575514.006323:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575514.006326:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575514.006330:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575514.006338:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393215104:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575514.006344:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006347:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575514.006350:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393215104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006356:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575514.006358:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575514.006361:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575514.006363:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575514.006365:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575514.006367:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575514.006369:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575514.006372:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575514.006375:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575514.006377:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575514.006379:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788575514.006382:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575514.006383:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006387:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006440:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575514.006443:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575514.006445:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575514.006446:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575514.006461:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575514.006465:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006471:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575514.006473:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575514.006475:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575514.006477:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575514.006479:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575514.006482:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393215232:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575514.006485:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575514.006487:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575514.006489:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575514.006490:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575514.006493:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575514.006494:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575514.006497:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575514.006499:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575514.006502:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575514.006506:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393215232, portal 10
00000100:00000001:2.0:1788575514.006507:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575514.006509:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:2.0:1788575514.006515:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006522:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575514.006524:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575514.006525:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575514.006530:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393215232, offset 0
00000400:00000200:2.0:1788575514.006534:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575514.006538:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575514.006542:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575514.006544:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575514.006547:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575514.006549:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575514.006551:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575514.006554:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330937:330937:256:4294967295] 192.168.202.110@tcp LPNI seq info [330894:330894:8:4294967295]
00000400:00000200:2.0:1788575514.006558:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575514.006565:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575514.006569:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575514.006572:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575514.006576:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575514.006579:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575514.006583:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575514.006586:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575514.006589:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575514.006591:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575514.006594:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575514.006597:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575514.006601:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575514.006602:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575514.006604:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575514.006608:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006614:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575514.006617:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575539 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006627:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575514.006628:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575514.006631:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575514.006636:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575514.006638:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575514.006641:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307117607
00000100:00000001:2.0:1788575514.006644:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307118521
00000100:00000001:2.0:1788575514.006646:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307117607
00000100:00000001:2.0:1788575514.006648:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307118521
00000100:00000001:2.0:1788575514.006650:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575514.006652:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575514.006653:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575514.006657:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575514/real 1788575514]  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575514.006663:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575514.006665:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575514.006666:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575514.006669:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575514.006670:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575514.006672:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575514.006675:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575514.006682:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006688:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575514.006689:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006691:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575514.006693:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575514.006696:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006702:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575514.006706:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006711:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575514.006713:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006715:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575514.006716:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006717:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575514.006720:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575514.006722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575514.006726:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575514.006728:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575514.006731:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575514.006739:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393215232:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575514.006742:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575514.006745:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006751:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575514.006752:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575514.006754:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575514.006756:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575514.006758:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575514.006761:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393215360:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575514.006763:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575514.006765:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575514.006767:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575514.006769:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575514.006772:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575514.006773:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575514.006776:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575514.006779:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575514.006782:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575514.006784:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393215360, portal 4
00000100:00000001:2.0:1788575514.006786:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575514.006787:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:2.0:1788575514.006793:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006798:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575514.006801:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575514.006802:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575514.006804:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393215360, offset 0
00000400:00000200:2.0:1788575514.006808:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575514.006812:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575514.006816:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575514.006818:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575514.006820:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575514.006822:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575514.006823:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575514.006825:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330938:330938:256:4294967295] 192.168.202.110@tcp LPNI seq info [330895:330895:8:4294967295]
00000400:00000200:2.0:1788575514.006829:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575514.006835:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575514.006839:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575514.006842:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575514.006845:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575514.006847:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575514.006851:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575514.006854:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575514.006857:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575514.006859:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575514.006862:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575514.006865:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575514.006868:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575514.006869:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575514.006872:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575514.006875:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006883:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575514.006885:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575539 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006892:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575514.006893:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575514.006897:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575514.006900:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575514.006902:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575514.006904:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307117753
00000100:00000001:2.0:1788575514.006906:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307118521
00000100:00000001:2.0:1788575514.006908:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307117753
00000100:00000001:2.0:1788575514.006910:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307118521
00000100:00000001:2.0:1788575514.006912:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575514.006914:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575514.006916:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575514.006920:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575514/real 1788575514]  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575514.006928:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575514.006929:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575514.006931:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575514.006933:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575514.006935:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575514.006937:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575514.006940:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575514.006948:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006954:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575514.006955:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006957:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575514.006958:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575514.006962:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006968:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575514.006972:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.006979:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575514.006981:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006983:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575514.006984:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.006986:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575514.006987:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575514.006990:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575514.006993:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575514.006995:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575514.006998:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575514.007005:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393215360:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575514.007008:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.007010:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575514.007014:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393215232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.007019:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575514.007020:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575514.007022:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575514.007025:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575514.007026:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575514.007027:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575514.007029:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575514.007031:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575514.007034:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575514.007035:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575514.007037:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:2.0:1788575514.007039:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575514.007040:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.007042:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575514.007046:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393215360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575539 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575514.007052:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575514.007053:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575514.007055:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575514.007056:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575514.007057:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575514.007058:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575514.007059:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575514.007060:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575514.007061:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575514.007062:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575514.007063:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788575514.007064:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575514.007065:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.007066:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575514.007069:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575514.007070:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575514.007071:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575514.007073:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575514.389464:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575514.389469:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575514.389472:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575514.389473:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575514.389475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575514.389476:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575514.389515:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575514.389519:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575514.389522:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575514.517448:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575514.517456:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575514.517459:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575514.517464:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575514.517468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575514.517481:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575514.517484:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575514.517486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575514.517489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575514.965488:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307118521, at 4307118528
00000100:00000040:1.0:1788575514.965488:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307118521, at 4307118528
00000100:00000010:0.0:1788575514.965501:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00000100:00000010:1.0:1788575514.965503:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00010000:00000001:3.0:1788575515.413550:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575515.413554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575515.413558:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575515.413560:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575515.413562:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575515.413565:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575515.413567:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575515.413568:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575515.413570:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575515.413571:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575515.413574:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575515.413575:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575515.413576:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575515.413577:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575515.413578:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575515.413579:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575515.413580:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575515.413581:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575515.413635:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575515.413637:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575515.413639:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575515.541470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575515.541477:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575515.541479:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575515.541484:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575515.541487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575515.541491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575515.541493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575515.541496:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575515.541498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575516.437466:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575516.437471:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575516.437474:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575516.437475:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575516.437478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575516.437479:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575516.437533:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575516.437536:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575516.437538:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575516.565507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575516.565514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575516.565515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575516.565518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575516.565520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575516.565523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575516.565525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575516.565526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575516.565529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575517.461473:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575517.461480:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575517.461482:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575517.461484:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575517.461487:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575517.461489:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575517.461490:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575517.461492:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575517.461494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575517.461495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575517.461497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575517.461498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575517.461544:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575517.461547:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575517.461549:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575517.589490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575517.589497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575517.589499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575517.589504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575517.589508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575517.589511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575517.589514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575517.589516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575517.589519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575518.485475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575518.485482:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575518.485487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575518.485489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575518.485500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575518.485501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575518.485553:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575518.485556:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575518.485559:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575518.613500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575518.613506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575518.613508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575518.613512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575518.613515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575518.613518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575518.613519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575518.613521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575518.613523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575519.125498:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575519.125507:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000800:00000200:0.0:1788575519.125509:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000100:00000001:3.0:1788575519.125510:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575519.125514:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575519.125518:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575519.125522:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12450
00000800:00000200:0.0:1788575519.125523:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000400:00000200:3.0:1788575519.125530:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:0.0:1788575519.125531:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000400:00000200:3.0:1788575519.125536:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575519.125539:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575519.125542:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125545:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575519.125547:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125549:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575519.125553:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575519.125554:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575519.125557:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575519.125560:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575519.125562:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575519.125563:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575519.125566:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125568:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575519.125570:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125572:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575519.125574:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575519.125578:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125581:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575519.125584:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575519.125586:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575519.125588:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575519.125590:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575519.125592:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575519.125600:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000020:00000040:3.0:1788575519.125604:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575519.125607:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575519.125609:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575519.125612:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575519.125615:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575519.125619:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575519.125626:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575519.125632:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575519.125648:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575519.125655:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575519.125658:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575519.125661:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575519.125676:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575519.125679:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575519.125683:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575519.125684:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575519.125686:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575519.125689:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575519.125692:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12450
00000400:00000200:3.0:1788575519.125697:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575519.125701:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575519.125703:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000001:0.0:1788575519.125706:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000040:3.0:1788575519.125706:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125708:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575519.125711:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125713:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575519.125715:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575519.125717:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575519.125718:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000001:0.0:1788575519.125721:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:1.0:1788575519.125721:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000040:3.0:1788575519.125721:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575519.125722:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575519.125725:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575519.125728:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575519.125729:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000800:00000001:0.0:1788575519.125730:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000040:3.0:1788575519.125733:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000001:0.0:1788575519.125735:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.125735:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575519.125736:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575519.125739:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125741:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575519.125746:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:2.0:1788575519.125747:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575519.125751:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125756:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575519.125759:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575519.125764:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1788575519.125765:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575519.125766:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575519.125770:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575519.125772:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575519.125774:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575519.125776:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:3.0:1788575519.125779:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575519.125780:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788575519.125783:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575519.125786:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575519.125788:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393215488:192.168.202.110@tcp:250:kworker.0
00000100:00000010:3.0:1788575519.125790:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575519.125793:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575519.125796:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575519.125797:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575519.125800:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575519.125801:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575519.125802:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575519.125803:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575519.125804:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575519.125807:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575519.125808:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575519.125811:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575519.125813:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575519.125819:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
00000100:00100000:3.0:1788575519.125819:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575519.125824:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575519.125829:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575519.125831:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575519.125836:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575519.125836:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575519.125843:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000001:3.0:1788575519.125844:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575519.125848:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575519.125851:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:2.0:1788575519.125852:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000001:3.0:1788575519.125852:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575519.125855:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575519.125858:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575519.125859:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12450
00000100:00000200:2.0:1788575519.125862:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393215488, portal 25
00000400:00000200:3.0:1788575519.125862:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575519.125867:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1788575519.125868:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1788575519.125869:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575519.125871:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:3.0:1788575519.125872:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125877:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575519.125878:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125880:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575519.125882:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:2.0:1788575519.125883:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575519.125883:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575519.125886:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575519.125888:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575519.125891:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575519.125895:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575519.125900:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575519.125903:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575519.125903:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575519.125908:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575519.125911:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125911:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575519.125913:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:2.0:1788575519.125916:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00080000:3.0:1788575519.125917:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125921:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575519.125922:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:2.0:1788575519.125923:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393215488, offset 0
02000000:00000001:3.0:1788575519.125925:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575519.125926:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575519.125927:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575519.125929:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575519.125930:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575519.125931:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575519.125934:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788575519.125938:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575519.125939:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575519.125940:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575519.125941:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575519.125942:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575519.125943:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575519.125946:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000400:00000200:2.0:1788575519.125947:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575519.125949:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575519.125953:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575519.125958:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575519.125960:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575519.125963:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575519.125966:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575519.125968:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788575519.125977:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575519.125979:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1788575519.125980:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575519.125985:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575519.125987:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1788575519.125990:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12460)
00000400:00000200:2.0:1788575519.125992:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575519.125996:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575519.126000:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330939:330939:256:4294967295] 192.168.202.110@tcp LPNI seq info [330896:330896:8:4294967295]
00000400:00000200:2.0:1788575519.126005:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575519.126014:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575519.126020:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575519.126024:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575519.126028:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575519.126031:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575519.126036:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575519.126038:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575519.126042:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575519.126044:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575519.126048:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575519.126052:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575519.126057:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575519.126059:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575519.126062:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575519.126066:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126075:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575519.126078:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575544 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126084:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575519.126086:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575519.126091:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575519.126095:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575519.126097:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307123641
00000100:00000010:2.0:1788575519.126102:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c000.
00000100:00000001:2.0:1788575519.126108:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575519.126119:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307123641
00000100:00000010:2.0:1788575519.126123:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c840.
00000100:00000001:2.0:1788575519.126125:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575519.126129:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575519.126132:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575519.126135:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575519.126140:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575519/real 1788575519]  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575519.126149:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575519.126150:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575519.126152:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575519.126155:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575519.126157:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575519.126159:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575519.126162:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575519.126170:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126176:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575519.126179:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126181:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575519.126183:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575519.126187:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126193:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575519.126197:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126204:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575519.126207:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126208:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575519.126209:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126211:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575519.126213:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575519.126216:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575519.126219:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575519.126221:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575519.126225:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575519.126232:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393215488:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575519.126236:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126238:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575519.126241:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393215488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126247:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575519.126249:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575519.126251:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575519.126254:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575519.126256:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575519.126258:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575519.126260:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575519.126262:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575519.126274:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575519.126276:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575519.126278:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788575519.126281:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575519.126283:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126284:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126291:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575519.126293:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575519.126295:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575519.126296:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575519.126297:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575519.126301:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126308:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575519.126309:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575519.126311:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575519.126312:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575519.126315:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575519.126317:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393215616:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575519.126320:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575519.126334:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575519.126336:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575519.126337:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575519.126340:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575519.126342:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575519.126346:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575519.126349:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575519.126352:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575519.126355:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393215616, portal 10
00000100:00000001:2.0:1788575519.126357:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575519.126358:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:2.0:1788575519.126363:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126370:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575519.126373:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575519.126376:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575519.126378:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393215616, offset 0
00000400:00000200:2.0:1788575519.126381:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575519.126385:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575519.126389:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575519.126418:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575519.126421:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575519.126423:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575519.126424:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575519.126426:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330940:330940:256:4294967295] 192.168.202.110@tcp LPNI seq info [330897:330897:8:4294967295]
00000400:00000200:2.0:1788575519.126430:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575519.126436:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575519.126441:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575519.126444:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575519.126448:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575519.126451:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575519.126454:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575519.126457:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575519.126460:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575519.126462:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575519.126465:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575519.126469:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575519.126472:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575519.126474:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575519.126476:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575519.126479:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126487:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575519.126490:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575544 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126500:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575519.126502:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575519.126505:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575519.126508:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575519.126512:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575519.126514:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307122727
00000100:00000001:2.0:1788575519.126516:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307123641
00000100:00000001:2.0:1788575519.126518:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307122727
00000100:00000001:2.0:1788575519.126520:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307123641
00000100:00000001:2.0:1788575519.126522:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575519.126524:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575519.126525:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575519.126529:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575519/real 1788575519]  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575519.126536:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575519.126537:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575519.126538:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575519.126540:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575519.126541:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575519.126542:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575519.126544:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575519.126549:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126559:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575519.126560:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126561:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575519.126561:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575519.126564:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126568:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575519.126571:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126574:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575519.126576:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126577:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575519.126577:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126578:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575519.126580:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575519.126581:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575519.126583:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575519.126584:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575519.126586:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575519.126590:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393215616:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575519.126593:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575519.126595:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126598:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575519.126599:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575519.126600:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575519.126602:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575519.126603:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575519.126604:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393215744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575519.126606:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575519.126607:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575519.126609:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575519.126610:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575519.126612:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575519.126613:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575519.126615:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575519.126617:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575519.126619:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575519.126621:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393215744, portal 4
00000100:00000001:2.0:1788575519.126622:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575519.126623:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788575519.126626:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126630:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575519.126632:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575519.126633:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575519.126634:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393215744, offset 0
00000400:00000200:2.0:1788575519.126636:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575519.126639:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575519.126641:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575519.126643:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575519.126644:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575519.126646:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575519.126647:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575519.126649:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330941:330941:256:4294967295] 192.168.202.110@tcp LPNI seq info [330898:330898:8:4294967295]
00000400:00000200:2.0:1788575519.126651:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575519.126655:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575519.126658:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575519.126660:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575519.126662:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575519.126664:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575519.126666:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575519.126668:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575519.126670:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575519.126672:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575519.126674:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575519.126676:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575519.126678:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575519.126679:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575519.126680:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575519.126682:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126686:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575519.126688:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575544 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126692:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575519.126693:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575519.126694:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575519.126696:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575519.126698:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575519.126699:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307122873
00000100:00000001:2.0:1788575519.126701:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307123641
00000100:00000001:2.0:1788575519.126702:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307122873
00000100:00000001:2.0:1788575519.126703:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307123641
00000100:00000001:2.0:1788575519.126705:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575519.126706:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575519.126707:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575519.126709:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575519/real 1788575519]  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575519.126713:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575519.126713:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575519.126714:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575519.126715:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575519.126716:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575519.126718:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575519.126719:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575519.126724:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126727:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575519.126728:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126729:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575519.126729:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575519.126731:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126735:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575519.126737:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126740:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575519.126741:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126742:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575519.126743:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126744:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575519.126745:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575519.126746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575519.126748:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575519.126749:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575519.126751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575519.126756:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393215744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575519.126758:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126759:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575519.126761:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393215616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126764:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575519.126765:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575519.126766:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575519.126768:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575519.126768:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575519.126769:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575519.126770:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575519.126771:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575519.126773:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575519.126774:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575519.126775:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575519.126776:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575519.126777:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126778:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575519.126780:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393215744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575544 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575519.126783:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575519.126784:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575519.126784:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575519.126785:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575519.126786:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575519.126787:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575519.126788:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575519.126788:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575519.126790:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575519.126790:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575519.126791:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575519.126792:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575519.126793:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126794:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575519.126796:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575519.126797:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575519.126798:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575519.126799:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575519.509509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575519.509514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575519.509520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575519.509521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575519.509524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575519.509525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575519.509580:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575519.509584:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575519.509587:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575519.637475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575519.637480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575519.637482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575519.637485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575519.637487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575519.637491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575519.637493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575519.637495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575519.637497:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575520.085467:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307123641, at 4307123648
00000100:00000040:1.0:1788575520.085470:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307123641, at 4307123648
00000100:00000010:0.0:1788575520.085482:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000100:00000010:1.0:1788575520.085484:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00010000:00000001:3.0:1788575520.533495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575520.533500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575520.533504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575520.533504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575520.533506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575520.533508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575520.533562:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575520.533566:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575520.533569:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575520.661536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575520.661543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575520.661545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575520.661550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575520.661553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575520.661556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575520.661558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575520.661559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575520.661561:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575521.557503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575521.557509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575521.557515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575521.557517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575521.557521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575521.557522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575521.557561:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575521.557564:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575521.557566:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575521.685483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575521.685489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575521.685495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575521.685500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575521.685504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575521.685507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575521.685510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575521.685513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575521.685516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575522.581516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575522.581522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575522.581528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575522.581529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575522.581532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575522.581534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575522.581593:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575522.581597:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575522.581600:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575522.709517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575522.709527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575522.709530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575522.709536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575522.709542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575522.709547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575522.709549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575522.709552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575522.709556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575523.605514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575523.605520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575523.605524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575523.605524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575523.605527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575523.605528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575523.605589:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575523.605592:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575523.605594:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575523.733523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575523.733529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575523.733530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575523.733534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575523.733536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575523.733539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575523.733541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575523.733543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575523.733545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575524.245538:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575524.245548:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575524.245551:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575524.245555:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575524.245560:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575524.245568:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12455
00000400:00000200:2.0:1788575524.245576:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575524.245583:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575524.245585:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575524.245589:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575524.245592:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575524.245594:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575524.245596:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575524.245600:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575524.245602:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575524.245604:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575524.245607:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575524.245609:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575524.245612:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575524.245615:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575524.245617:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575524.245619:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575524.245621:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575524.245624:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575524.245627:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575524.245630:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575524.245632:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575524.245635:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575524.245638:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575524.245640:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575524.245642:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575524.245651:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00700.
00000020:00000040:2.0:1788575524.245655:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575524.245659:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575524.245660:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575524.245663:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575524.245666:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575524.245669:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575524.245676:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:2.0:1788575524.245681:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575524.245689:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575524.245699:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575524.245702:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575524.245707:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e69941e] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575524.245726:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575524.245729:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575524.245733:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575524.245734:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575524.245736:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575524.245739:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575524.245742:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12455
00000400:00000200:2.0:1788575524.245747:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575524.245751:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1788575524.245753:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1788575524.245754:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575524.245760:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575524.245760:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575524.245764:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575524.245766:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575524.245768:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575524.245770:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1788575524.245771:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1788575524.245772:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575524.245776:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575524.245778:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575524.245781:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575524.245781:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575524.245786:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575524.245797:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575524.245799:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575524.245800:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575524.245803:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1788575524.245804:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1788575524.245807:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575524.245809:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575524.245812:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575524.245814:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575524.245818:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00100000:3.0:1788575524.245820:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393215872:192.168.202.110@tcp:250:kworker.0
00000100:00080000:2.0:1788575524.245824:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575524.245828:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1788575524.245829:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575524.245832:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575524.245834:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575524.245838:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575524.245838:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575524.245843:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575524.245845:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788575524.245847:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:3.0:1788575524.245851:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
00000008:00100000:2.0:1788575524.245852:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575524.245856:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788575524.245857:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575524.245861:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575524.245866:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02680.
00000400:00000200:3.0:1788575524.245866:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575524.245871:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:3.0:1788575524.245872:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
02000000:00000001:2.0:1788575524.245875:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575524.245878:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575524.245880:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575524.245881:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1788575524.245881:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000001:2.0:1788575524.245884:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575524.245890:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393215872, portal 25
02000000:00000010:2.0:1788575524.245891:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:3.0:1788575524.245896:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575524.245897:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.245899:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00100000:2.0:1788575524.245906:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1788575524.245909:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575524.245931:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575524.245931:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1788575524.245937:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575524.245937:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575524.245942:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000040:2.0:1788575524.245947:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000450d1599] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575524.245948:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393215872, offset 0
00000400:00000200:3.0:1788575524.245960:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575524.245961:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575524.245964:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575524.245968:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575524.245971:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1788575524.245971:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575524.245976:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575524.245979:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:3.0:1788575524.245981:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1788575524.245984:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12455
00000400:00000200:3.0:1788575524.245989:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575524.245993:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575524.245996:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575524.245998:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575524.246001:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330942:330942:256:4294967295] 192.168.202.110@tcp LPNI seq info [330899:330899:8:4294967295]
00000400:00000200:3.0:1788575524.246004:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575524.246009:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575524.246015:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1788575524.246019:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575524.246023:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1788575524.246024:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575524.246029:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000010:3.0:1788575524.246030:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000100:00000001:2.0:1788575524.246033:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575524.246040:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575524.246041:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575524.246043:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:3.0:1788575524.246045:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1788575524.246047:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575524.246050:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:3.1:1788575524.246051:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1788575524.246053:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.1:1788575524.246055:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000040:2.0:1788575524.246056:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575524.246061:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000100:3.0:1788575524.246061:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575524.246063:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000020:00000001:2.0:1788575524.246066:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575524.246069:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1788575524.246071:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575524.246076:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575524.246076:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575524.246080:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000100:00000040:2.0:1788575524.246081:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:3.0:1788575524.246084:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:2.0:1788575524.246086:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575524.246090:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:3.0:1788575524.246090:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00080000:2.0:1788575524.246096:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:3.0:1788575524.246097:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575524.246101:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575524.246106:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575524.246110:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575524.246111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1788575524.246115:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575524.246116:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575549 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575524.246120:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575524.246124:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:3.0:1788575524.246125:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246128:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000008:00000001:2.0:1788575524.246130:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575524.246133:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246134:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575524.246138:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02a00.
00000400:00000200:3.0:1788575524.246138:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575524.246142:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307128761
00000020:00000040:2.0:1788575524.246143:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575524.246148:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000010:3.0:1788575524.246148:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
02000000:00000001:2.0:1788575524.246149:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575524.246154:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575524.246155:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1788575524.246156:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:2.0:1788575524.246158:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246162:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307128761
02000000:00000010:2.0:1788575524.246165:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000010:3.0:1788575524.246166:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575524.246170:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575524.246171:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246173:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575524.246176:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246178:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575524.246181:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1788575524.246184:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575524/real 1788575524]  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575524.246208:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000001:3.0:1788575524.246210:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1788575524.246213:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1788575524.246213:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575524.246216:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575524.246218:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575524.246221:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000040:2.0:1788575524.246222:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cce910e4] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246223:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575524.246229:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575524.246245:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575524.246248:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575524.246250:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575524.246258:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575524.246261:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575524.246264:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575524.246265:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12465)
00000100:00000001:3.0:1788575524.246266:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575524.246270:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575524.246273:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246277:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575524.246280:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246285:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575524.246287:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246288:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575524.246288:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246290:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575524.246291:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575524.246293:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575524.246295:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575524.246296:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575524.246298:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575524.246303:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393215872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575524.246306:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246307:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575524.246309:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393215872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246313:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575524.246315:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575524.246317:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575524.246319:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575524.246320:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575524.246321:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575524.246323:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575524.246324:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575524.246327:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575524.246328:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575524.246329:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575524.246331:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575524.246332:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246333:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246338:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575524.246339:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246340:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575524.246341:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575524.246342:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575524.246344:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246348:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575524.246349:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575524.246350:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575524.246352:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575524.246353:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575524.246354:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575524.246356:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575524.246357:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575524.246358:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575524.246359:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575524.246360:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575524.246361:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575524.246364:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575524.246365:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575524.246367:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575524.246369:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393216000, portal 10
00000100:00000001:3.0:1788575524.246370:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575524.246371:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:3.0:1788575524.246375:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246379:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575524.246380:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575524.246382:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575524.246383:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393216000, offset 0
00000400:00000200:3.0:1788575524.246385:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575524.246387:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575524.246389:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575524.246408:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575524.246410:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575524.246411:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575524.246413:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575524.246416:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330943:330943:256:4294967295] 192.168.202.110@tcp LPNI seq info [330900:330900:8:4294967295]
00000400:00000200:3.0:1788575524.246420:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575524.246425:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575524.246430:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575524.246433:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575524.246436:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575524.246439:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788575524.246443:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575524.246445:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1788575524.246449:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575524.246451:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575524.246465:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575524.246469:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575524.246472:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575524.246473:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575524.246475:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575524.246478:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246486:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575524.246488:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575549 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246492:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246493:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575524.246495:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575524.246497:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575524.246499:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575524.246501:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307127847
00000100:00000001:3.0:1788575524.246502:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307128761
00000100:00000001:3.0:1788575524.246504:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307127847
00000100:00000001:3.0:1788575524.246505:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307128761
00000100:00000001:3.0:1788575524.246506:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575524.246508:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246509:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575524.246511:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575524/real 1788575524]  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575524.246515:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575524.246516:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575524.246516:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575524.246518:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575524.246519:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575524.246520:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575524.246521:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575524.246526:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246529:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575524.246531:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246532:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575524.246533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575524.246535:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246539:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575524.246541:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246544:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575524.246546:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246546:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575524.246547:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246548:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575524.246549:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575524.246551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575524.246552:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575524.246554:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575524.246556:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575524.246560:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575524.246562:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575524.246564:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246568:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575524.246569:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575524.246570:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575524.246571:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575524.246572:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575524.246574:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216128:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575524.246576:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575524.246577:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575524.246579:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575524.246580:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575524.246582:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575524.246583:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575524.246585:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575524.246587:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575524.246588:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575524.246590:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393216128, portal 4
00000100:00000001:3.0:1788575524.246591:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575524.246592:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:3.0:1788575524.246596:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246600:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575524.246601:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575524.246602:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575524.246604:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393216128, offset 0
00000400:00000200:3.0:1788575524.246607:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575524.246610:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575524.246614:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575524.246616:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575524.246619:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575524.246621:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575524.246623:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575524.246626:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330944:330944:256:4294967295] 192.168.202.110@tcp LPNI seq info [330901:330901:8:4294967295]
00000400:00000200:3.0:1788575524.246630:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575524.246636:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575524.246640:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575524.246643:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575524.246647:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575524.246650:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788575524.246654:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575524.246655:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1788575524.246658:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575524.246659:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575524.246661:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575524.246663:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575524.246665:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575524.246666:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575524.246667:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575524.246670:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246675:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575524.246676:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575549 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246680:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246681:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575524.246683:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575524.246685:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575524.246687:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575524.246688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307127993
00000100:00000001:3.0:1788575524.246690:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307128761
00000100:00000001:3.0:1788575524.246691:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307127993
00000100:00000001:3.0:1788575524.246692:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307128761
00000100:00000001:3.0:1788575524.246693:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575524.246694:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246695:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575524.246697:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575524/real 1788575524]  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575524.246702:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575524.246703:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575524.246704:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575524.246705:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575524.246706:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575524.246708:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575524.246709:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575524.246715:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246720:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575524.246721:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246723:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575524.246724:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575524.246727:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246733:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575524.246737:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246743:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575524.246745:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246747:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575524.246748:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246749:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575524.246751:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575524.246753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575524.246756:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575524.246758:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575524.246762:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575524.246776:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216128:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575524.246779:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246781:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575524.246784:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393216000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246787:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575524.246789:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575524.246790:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575524.246791:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575524.246793:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575524.246793:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575524.246795:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575524.246796:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575524.246797:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575524.246798:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575524.246799:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575524.246801:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575524.246802:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246803:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575524.246804:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393216128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575549 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575524.246807:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575524.246808:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575524.246809:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575524.246810:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575524.246811:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575524.246811:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575524.246812:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575524.246813:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575524.246814:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575524.246815:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575524.246815:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:3.0:1788575524.246817:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575524.246818:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246819:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575524.246821:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575524.246822:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575524.246823:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575524.246824:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575524.629485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575524.629489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575524.629492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575524.629493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575524.629495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575524.629495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575524.629546:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575524.629550:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575524.629553:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575524.757472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575524.757480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575524.757482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575524.757487:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575524.757489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575524.757492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575524.757494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575524.757496:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575524.757499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575525.205464:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307128761, at 4307128768
00000100:00000040:1.0:1788575525.205486:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307128761, at 4307128768
00000100:00000010:1.0:1788575525.205499:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788575525.205503:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575525.653467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575525.653473:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575525.653491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575525.653494:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575525.653496:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575525.653499:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575525.653502:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575525.653504:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575525.653506:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575525.653508:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575525.653512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575525.653513:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575525.653514:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575525.653516:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575525.653518:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575525.653519:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575525.653520:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575525.653522:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575525.653573:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575525.653577:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575525.653580:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575525.781515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575525.781523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575525.781525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575525.781530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575525.781533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575525.781537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575525.781539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575525.781543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575525.781546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575526.677509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575526.677515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575526.677520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575526.677520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575526.677523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575526.677523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575526.677591:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575526.677595:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575526.677598:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575526.805477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575526.805485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575526.805488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575526.805493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575526.805507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575526.805511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575526.805514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575526.805516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575526.805519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575527.701489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575527.701497:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575527.701500:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575527.701503:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575527.701518:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575527.701520:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575527.701522:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575527.701524:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575527.701528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575527.701530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575527.701533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575527.701534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575527.701594:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575527.701598:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575527.701600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575527.829493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575527.829499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575527.829501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575527.829506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575527.829537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575527.829540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575527.829543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575527.829545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575527.829548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575528.725506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575528.725513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575528.725518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575528.725520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575528.725537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575528.725539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575528.725583:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575528.725587:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575528.725590:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575528.853472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575528.853480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575528.853482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575528.853487:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575528.853490:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575528.853494:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575528.853496:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575528.853499:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575528.853502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575529.365492:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575529.365500:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575529.365503:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575529.365506:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575529.365510:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575529.365515:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12460
00000400:00000200:2.0:1788575529.365521:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575529.365531:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575529.365554:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575529.365557:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365560:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575529.365562:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365564:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575529.365567:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575529.365568:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575529.365570:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575529.365573:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575529.365576:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575529.365578:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575529.365580:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365581:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575529.365582:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365584:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575529.365586:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575529.365588:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365589:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575529.365591:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575529.365593:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575529.365595:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575529.365596:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575529.365598:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575529.365605:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03800.
00000020:00000040:2.0:1788575529.365607:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575529.365610:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575529.365611:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575529.365613:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575529.365615:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575529.365618:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575529.365624:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12c00.
00000100:00000001:2.0:1788575529.365629:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575529.365635:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575529.365646:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575529.365649:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575529.365652:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014d98894] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575529.365669:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575529.365672:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575529.365675:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575529.365676:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575529.365678:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575529.365681:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575529.365683:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12460
00000400:00000200:2.0:1788575529.365687:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575529.365690:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575529.365692:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575529.365694:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365696:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575529.365698:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365701:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575529.365703:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575529.365704:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575529.365705:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575529.365706:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1788575529.365707:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575529.365709:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575529.365710:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1788575529.365711:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575529.365712:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575529.365714:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365715:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575529.365717:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575529.365718:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575529.365719:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575529.365722:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575529.365725:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575529.365726:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575529.365727:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575529.365728:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575529.365729:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575529.365731:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575529.365733:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575529.365733:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575529.365735:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575529.365735:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575529.365737:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:3.0:1788575529.365739:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393216256:192.168.202.110@tcp:250:kworker.0
00000008:00100000:2.0:1788575529.365741:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575529.365742:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1788575529.365744:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575529.365746:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575529.365746:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575529.365748:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575529.365749:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01f80.
02000000:00000001:3.0:1788575529.365750:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1788575529.365751:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575529.365754:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575529.365755:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575529.365755:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
00000100:00000001:2.0:1788575529.365756:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575529.365757:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575529.365758:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575529.365759:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575529.365762:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11400.
00000400:00000200:3.0:1788575529.365762:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575529.365765:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:2.0:1788575529.365766:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575529.365770:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575529.365772:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575529.365776:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575529.365778:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575529.365778:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393216256, portal 25
00000100:00000040:2.0:1788575529.365781:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8c8ec7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.365781:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575529.365783:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:3.0:1788575529.365788:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575529.365789:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575529.365791:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575529.365794:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575529.365794:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1788575529.365796:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575529.365797:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:3.0:1788575529.365797:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365800:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:3.0:1788575529.365800:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00080000:2.0:1788575529.365803:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12460
00000100:00000200:3.0:1788575529.365803:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393216256, offset 0
00000400:00000200:2.0:1788575529.365806:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575529.365809:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1788575529.365809:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575529.365811:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575529.365813:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365815:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575529.365816:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575529.365817:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365818:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575529.365821:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575529.365822:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:3.0:1788575529.365822:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:2.0:1788575529.365823:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575529.365825:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575529.365826:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575529.365827:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575529.365830:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575529.365834:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1788575529.365836:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575529.365836:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:2.0:1788575529.365839:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365841:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575529.365841:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330945:330945:256:4294967295] 192.168.202.110@tcp LPNI seq info [330902:330902:8:4294967295]
00000100:00000040:2.0:1788575529.365843:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365845:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575529.365845:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575529.365847:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575529.365849:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575529.365851:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575529.365853:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575529.365854:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1788575529.365856:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575529.365857:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575529.365859:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.0:1788575529.365860:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00100000:2.0:1788575529.365861:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575529.365863:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575529.365864:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575529.365864:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000100:00000010:2.0:1788575529.365867:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00a80.
00000020:00000040:2.0:1788575529.365869:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575529.365871:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.0:1788575529.365871:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1788575529.365873:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575529.365874:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:3.0:1788575529.365875:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
02000000:00000001:2.0:1788575529.365876:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575529.365877:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575529.365880:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000800:00000200:3.1:1788575529.365880:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575529.365883:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575529.365884:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00100000:2.0:1788575529.365887:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:3.0:1788575529.365889:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575529.365891:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000100:00000001:2.0:1788575529.365893:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575529.365895:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575529.365897:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1788575529.365898:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000389a8694] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575529.365902:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1788575529.365905:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575529.365908:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575529.365908:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575529.365910:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000040:2.0:1788575529.365911:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575529.365914:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:2.0:1788575529.365915:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12470)
00000100:00000200:3.0:1788575529.365917:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.365925:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575529.365928:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.365935:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575529.365937:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575529.365942:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575529.365947:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575529.365949:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307133881
00000100:00000010:3.0:1788575529.365955:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575529.365960:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575529.365972:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307133881
00000100:00000010:3.0:1788575529.365975:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575529.365977:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575529.365979:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575529.365981:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575529.365983:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575529.365987:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575529/real 1788575529]  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575529.365994:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575529.365996:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575529.365997:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575529.365999:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575529.366000:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575529.366002:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575529.366004:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575529.366009:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366012:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575529.366014:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366016:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575529.366017:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575529.366019:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366023:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575529.366026:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366030:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575529.366032:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366033:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575529.366033:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366035:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575529.366036:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575529.366038:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575529.366040:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575529.366042:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575529.366044:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575529.366048:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393216256:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575529.366051:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366052:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575529.366054:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393216256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366058:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575529.366060:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575529.366062:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575529.366064:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575529.366065:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575529.366066:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575529.366068:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575529.366070:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575529.366072:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575529.366074:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575529.366075:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:3.0:1788575529.366077:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575529.366077:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366079:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366084:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575529.366085:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575529.366086:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575529.366087:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575529.366088:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575529.366090:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366094:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575529.366095:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575529.366096:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575529.366098:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575529.366099:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575529.366100:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216384:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575529.366102:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575529.366103:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575529.366104:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575529.366105:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575529.366107:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575529.366108:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575529.366110:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575529.366112:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575529.366114:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575529.366116:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393216384, portal 10
00000100:00000001:3.0:1788575529.366117:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575529.366118:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:3.0:1788575529.366122:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366126:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575529.366128:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575529.366129:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575529.366130:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393216384, offset 0
00000400:00000200:3.0:1788575529.366133:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575529.366135:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575529.366137:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575529.366139:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575529.366140:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575529.366142:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575529.366143:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575529.366145:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330946:330946:256:4294967295] 192.168.202.110@tcp LPNI seq info [330903:330903:8:4294967295]
00000400:00000200:3.0:1788575529.366148:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575529.366152:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575529.366155:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575529.366157:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575529.366160:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575529.366162:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575529.366164:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575529.366166:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575529.366168:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575529.366169:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575529.366171:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575529.366173:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575529.366175:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575529.366176:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575529.366177:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575529.366179:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366189:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575529.366191:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366199:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575529.366200:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575529.366202:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575529.366204:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575529.366206:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575529.366207:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307132966
00000100:00000001:3.0:1788575529.366209:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307133881
00000100:00000001:3.0:1788575529.366211:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307132966
00000100:00000001:3.0:1788575529.366212:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307133881
00000100:00000001:3.0:1788575529.366213:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575529.366214:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575529.366215:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575529.366217:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575529/real 1788575529]  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575529.366221:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575529.366222:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575529.366223:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575529.366225:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575529.366226:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575529.366227:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575529.366228:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575529.366233:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366236:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575529.366237:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366238:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575529.366239:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575529.366241:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366245:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575529.366253:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366257:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575529.366259:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366260:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575529.366261:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366261:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575529.366263:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575529.366264:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575529.366266:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575529.366267:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575529.366269:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575529.366273:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216384:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575529.366275:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575529.366277:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366281:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575529.366282:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575529.366283:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575529.366284:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575529.366285:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575529.366286:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216512:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575529.366288:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575529.366289:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575529.366290:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575529.366291:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575529.366293:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575529.366294:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575529.366296:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575529.366298:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575529.366300:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575529.366301:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393216512, portal 4
00000100:00000001:3.0:1788575529.366302:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575529.366303:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:3.0:1788575529.366307:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366310:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575529.366312:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575529.366313:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575529.366314:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393216512, offset 0
00000400:00000200:3.0:1788575529.366316:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575529.366318:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575529.366320:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575529.366322:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575529.366323:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575529.366324:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575529.366326:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575529.366328:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330947:330947:256:4294967295] 192.168.202.110@tcp LPNI seq info [330904:330904:8:4294967295]
00000400:00000200:3.0:1788575529.366330:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575529.366334:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575529.366336:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575529.366338:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575529.366340:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575529.366341:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575529.366344:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575529.366345:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575529.366348:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575529.366349:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575529.366351:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575529.366352:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575529.366354:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575529.366355:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575529.366356:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575529.366358:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366363:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575529.366364:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366368:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575529.366369:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575529.366370:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575529.366372:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575529.366373:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575529.366375:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307133112
00000100:00000001:3.0:1788575529.366376:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307133881
00000100:00000001:3.0:1788575529.366377:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307133112
00000100:00000001:3.0:1788575529.366378:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307133881
00000100:00000001:3.0:1788575529.366380:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575529.366381:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575529.366382:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575529.366384:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575529/real 1788575529]  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575529.366387:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575529.366388:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575529.366389:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575529.366411:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575529.366412:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575529.366413:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575529.366415:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575529.366420:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366424:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575529.366425:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366426:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575529.366427:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575529.366428:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366438:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575529.366440:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366444:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575529.366446:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366447:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575529.366448:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366448:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575529.366449:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575529.366451:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575529.366452:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575529.366454:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575529.366456:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575529.366460:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216512:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575529.366462:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366463:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575529.366465:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393216384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366469:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575529.366470:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575529.366471:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575529.366472:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575529.366473:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575529.366474:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575529.366475:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575529.366476:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575529.366478:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575529.366479:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575529.366480:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575529.366481:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575529.366482:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366483:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575529.366485:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393216512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575529.366488:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575529.366489:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575529.366489:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575529.366491:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575529.366491:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575529.366492:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575529.366493:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575529.366493:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575529.366494:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575529.366495:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575529.366496:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575529.366497:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575529.366497:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366498:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575529.366500:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575529.366501:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575529.366502:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575529.366503:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575529.749576:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575529.749583:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575529.749588:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575529.749589:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575529.749592:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575529.749593:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575529.749669:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575529.749674:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575529.749677:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575529.877516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575529.877526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575529.877528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575529.877535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575529.877539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575529.877559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575529.877562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575529.877565:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575529.877567:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575530.325539:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307133881, at 4307133888
00000100:00000040:1.0:1788575530.325539:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307133881, at 4307133888
00000100:00000010:1.0:1788575530.325568:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788575530.325571:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575530.773552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575530.773569:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575530.773574:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575530.773575:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575530.773578:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575530.773579:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575530.773638:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575530.773641:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575530.773645:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575530.901526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575530.901535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575530.901537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575530.901542:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575530.901546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575530.901550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575530.901552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575530.901554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575530.901556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575531.797455:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575531.797460:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575531.797464:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575531.797465:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575531.797467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575531.797468:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575531.797533:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575531.797539:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575531.797544:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575531.925492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575531.925497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575531.925499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575531.925503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575531.925505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575531.925508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575531.925510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575531.925511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575531.925513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575532.821522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575532.821527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575532.821531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575532.821532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575532.821535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575532.821536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575532.821586:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575532.821614:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575532.821617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575532.949501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575532.949508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575532.949510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575532.949514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575532.949517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575532.949519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575532.949521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575532.949523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575532.949525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575533.845511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575533.845517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575533.845528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575533.845529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575533.845532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575533.845532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575533.845578:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575533.845582:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575533.845584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575533.973502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575533.973508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575533.973510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575533.973514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575533.973516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575533.973520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575533.973522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575533.973523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575533.973526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575534.485514:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575534.485522:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575534.485524:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575534.485527:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575534.485530:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575534.485536:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12465
00000400:00000200:0.0:1788575534.485543:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575534.485547:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575534.485549:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575534.485552:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485554:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575534.485555:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485557:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575534.485560:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575534.485562:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575534.485564:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575534.485566:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575534.485568:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575534.485569:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575534.485571:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485572:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575534.485573:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485574:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575534.485576:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575534.485579:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485582:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575534.485584:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575534.485587:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575534.485589:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575534.485591:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575534.485592:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575534.485599:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788575534.485601:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575534.485604:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575534.485606:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575534.485608:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575534.485610:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575534.485613:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575534.485631:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788575534.485637:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575534.485644:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575534.485650:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575534.485653:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575534.485656:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575534.485671:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575534.485673:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575534.485675:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575534.485676:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575534.485677:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575534.485679:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575534.485680:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12465
00000400:00000200:0.0:1788575534.485683:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575534.485684:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575534.485686:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575534.485688:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485689:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575534.485690:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485692:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575534.485693:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575534.485694:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575534.485695:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575534.485700:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575534.485702:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575534.485703:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575534.485705:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485707:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575534.485709:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485710:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575534.485712:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575534.485713:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485714:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575534.485715:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575534.485718:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:0.0:1788575534.485720:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575534.485723:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575534.485724:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575534.485724:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000008:00100000:0.0:1788575534.485728:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575534.485729:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000008:00000001:0.0:1788575534.485731:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575534.485733:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575534.485735:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575534.485739:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000040:3.0:1788575534.485740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575534.485741:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575534.485743:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575534.485746:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575534.485748:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575534.485749:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575534.485753:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788575534.485758:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.485759:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00100000:0.0:1788575534.485763:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575534.485763:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575534.485771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575534.485775:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575534.485776:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575534.485780:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575534.485780:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575534.485783:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575534.485789:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393216640:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575534.485793:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575534.485797:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575534.485797:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:0.0:1788575534.485799:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
02000000:00000001:3.0:1788575534.485803:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1788575534.485805:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575534.485807:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:3.0:1788575534.485807:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575534.485812:0:188751:0:(import.c:520:import_select_connection()) Process entered
02000000:00000001:3.0:1788575534.485814:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:0.0:1788575534.485815:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12465
00000400:00000200:0.0:1788575534.485819:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575534.485820:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575534.485822:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
02000000:00000010:3.0:1788575534.485823:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
00000100:00000040:0.0:1788575534.485824:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485829:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1788575534.485829:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575534.485831:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485834:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575534.485835:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575534.485836:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575534.485837:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575534.485839:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575534.485839:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1788575534.485842:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575534.485844:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575534.485847:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:3.0:1788575534.485847:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:0.0:1788575534.485850:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575534.485853:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485855:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575534.485856:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:3.0:1788575534.485857:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00080000:0.0:1788575534.485858:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485862:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575534.485863:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575534.485865:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575534.485867:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575534.485868:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:3.0:1788575534.485868:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393216640, portal 25
00000008:00100000:0.0:1788575534.485870:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575534.485873:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.485874:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575534.485875:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.485879:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
00000100:00000010:0.0:1788575534.485880:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788575534.485883:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575534.485886:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575534.485887:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575534.485889:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575534.485891:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1788575534.485891:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575534.485893:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575534.485897:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788575534.485901:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575534.485907:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575534.485908:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788575534.485915:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1788575534.485916:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575534.485919:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575534.485922:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000040:0.0:1788575534.485923:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575534.485930:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393216640, offset 0
00000100:00000001:0.0:1788575534.485931:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575534.485934:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575534.485937:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575534.485941:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12475)
00000400:00000200:3.0:1788575534.485943:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575534.485953:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575534.485960:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575534.485965:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575534.485968:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575534.485971:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575534.485974:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575534.485978:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330948:330948:256:4294967295] 192.168.202.110@tcp LPNI seq info [330905:330905:8:4294967295]
00000400:00000200:3.0:1788575534.485982:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575534.485991:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575534.485997:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575534.486002:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575534.486008:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575534.486012:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788575534.486017:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575534.486020:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1788575534.486024:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575534.486026:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575534.486031:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575534.486036:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575534.486040:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575534.486042:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575534.486047:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575534.486051:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486060:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575534.486063:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486070:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.486072:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575534.486078:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575534.486082:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575534.486085:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307139001
00000100:00000010:3.0:1788575534.486089:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575534.486096:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575534.486109:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307139001
00000100:00000010:3.0:1788575534.486112:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575534.486116:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575534.486119:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575534.486122:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.486125:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575534.486130:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575534/real 1788575534]  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575534.486145:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575534.486147:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575534.486148:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575534.486151:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575534.486152:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575534.486154:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575534.486157:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575534.486165:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486170:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575534.486171:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486173:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575534.486176:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575534.486179:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486185:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575534.486189:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486195:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575534.486198:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486200:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575534.486201:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486204:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575534.486206:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575534.486208:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575534.486211:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575534.486213:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575534.486217:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575534.486224:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393216640:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575534.486228:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486230:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575534.486233:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393216640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486238:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575534.486240:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575534.486242:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575534.486245:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575534.486248:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575534.486249:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575534.486251:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575534.486253:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:3.0:1788575534.486256:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575534.486257:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575534.486259:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:3.0:1788575534.486262:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575534.486263:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486265:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486271:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575534.486273:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.486275:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575534.486276:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575534.486277:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575534.486280:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486286:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575534.486287:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575534.486289:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575534.486291:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575534.486292:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575534.486295:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575534.486298:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575534.486299:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575534.486300:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575534.486302:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575534.486304:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575534.486306:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575534.486309:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575534.486311:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575534.486314:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575534.486317:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393216768, portal 10
00000100:00000001:3.0:1788575534.486318:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575534.486319:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000040:3.0:1788575534.486326:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486332:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575534.486334:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575534.486336:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575534.486338:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393216768, offset 0
00000400:00000200:3.0:1788575534.486342:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575534.486345:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575534.486349:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575534.486351:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575534.486354:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575534.486356:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575534.486358:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575534.486361:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330949:330949:256:4294967295] 192.168.202.110@tcp LPNI seq info [330906:330906:8:4294967295]
00000400:00000200:3.0:1788575534.486365:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575534.486370:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575534.486375:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575534.486377:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575534.486381:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575534.486384:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788575534.486388:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575534.486408:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1788575534.486424:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575534.486426:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575534.486429:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575534.486433:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575534.486436:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575534.486437:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575534.486439:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575534.486443:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486449:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575534.486452:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486458:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.486460:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575534.486463:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575534.486466:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575534.486469:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575534.486471:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307138087
00000100:00000001:3.0:1788575534.486474:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307139001
00000100:00000001:3.0:1788575534.486476:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307138087
00000100:00000001:3.0:1788575534.486477:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307139001
00000100:00000001:3.0:1788575534.486479:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575534.486481:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.486483:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575534.486487:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575534/real 1788575534]  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575534.486494:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575534.486495:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575534.486497:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575534.486499:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575534.486501:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575534.486503:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575534.486506:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575534.486515:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486521:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575534.486523:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575534.486526:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575534.486529:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486536:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575534.486539:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486545:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575534.486548:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486549:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575534.486550:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486552:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575534.486553:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575534.486556:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575534.486558:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575534.486560:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575534.486562:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575534.486567:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575534.486570:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575534.486572:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486576:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575534.486577:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575534.486578:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575534.486579:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575534.486580:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575534.486582:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575534.486584:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575534.486585:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575534.486586:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575534.486587:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575534.486589:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575534.486591:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575534.486593:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575534.486595:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575534.486597:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575534.486599:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393216896, portal 4
00000100:00000001:3.0:1788575534.486600:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575534.486601:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:3.0:1788575534.486604:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486608:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575534.486609:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575534.486611:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575534.486612:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393216896, offset 0
00000400:00000200:3.0:1788575534.486628:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575534.486631:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575534.486633:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575534.486635:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575534.486637:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575534.486638:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575534.486640:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575534.486642:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330950:330950:256:4294967295] 192.168.202.110@tcp LPNI seq info [330907:330907:8:4294967295]
00000400:00000200:3.0:1788575534.486644:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575534.486647:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575534.486650:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575534.486652:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575534.486654:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575534.486656:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788575534.486659:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575534.486660:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1788575534.486662:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575534.486664:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575534.486666:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575534.486668:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575534.486669:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575534.486670:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575534.486672:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575534.486674:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486679:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575534.486681:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486684:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.486685:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575534.486687:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575534.486689:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575534.486691:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575534.486692:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307138233
00000100:00000001:3.0:1788575534.486693:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307139001
00000100:00000001:3.0:1788575534.486695:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307138233
00000100:00000001:3.0:1788575534.486696:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307139001
00000100:00000001:3.0:1788575534.486697:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575534.486699:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.486700:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575534.486702:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575534/real 1788575534]  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575534.486706:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575534.486707:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575534.486708:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575534.486709:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575534.486710:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575534.486711:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575534.486712:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575534.486717:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486721:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575534.486722:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486722:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575534.486724:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575534.486727:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486733:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575534.486736:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486742:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575534.486744:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486746:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575534.486747:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486749:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575534.486751:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575534.486753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575534.486756:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575534.486758:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575534.486762:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575534.486770:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393216896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575534.486773:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486775:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575534.486778:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393216768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486784:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575534.486785:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575534.486787:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575534.486790:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575534.486792:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575534.486793:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575534.486795:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575534.486796:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:3.0:1788575534.486799:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575534.486800:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575534.486801:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:3.0:1788575534.486804:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575534.486805:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486806:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575534.486810:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393216896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575534.486816:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575534.486817:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575534.486818:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575534.486821:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575534.486822:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575534.486823:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575534.486825:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575534.486826:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:3.0:1788575534.486828:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575534.486829:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575534.486830:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:3.0:1788575534.486831:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575534.486832:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486833:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575534.486836:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575534.486836:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575534.486837:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575534.486838:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575534.869502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575534.869508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575534.869513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575534.869516:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575534.869518:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575534.869521:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575534.869524:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575534.869526:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575534.869528:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575534.869530:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575534.869533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575534.869534:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575534.869536:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575534.869537:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575534.869539:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575534.869540:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575534.869542:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575534.869543:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575534.869595:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575534.869598:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575534.869602:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575534.997481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575534.997486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575534.997488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575534.997492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575534.997494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575534.997497:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575534.997499:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575534.997501:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575534.997502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575535.445474:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307139001, at 4307139008
00000100:00000040:1.0:1788575535.445475:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307139001, at 4307139008
00000100:00000010:0.0:1788575535.445489:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:1.0:1788575535.445492:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00010000:00000001:3.0:1788575535.893487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575535.893494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575535.893501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575535.893503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575535.893506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575535.893507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575535.893545:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575535.893548:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575535.893550:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575536.021493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575536.021498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575536.021499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575536.021503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575536.021505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575536.021508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575536.021509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575536.021511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575536.021513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575536.917500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575536.917506:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575536.917508:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575536.917510:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575536.917514:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575536.917516:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575536.917517:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575536.917518:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575536.917520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575536.917521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575536.917524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575536.917524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575536.917576:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575536.917580:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575536.917583:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575537.045475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575537.045480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575537.045482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575537.045486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575537.045489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575537.045492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575537.045494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575537.045497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575537.045500:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575537.941458:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575537.941462:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575537.941465:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575537.941466:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575537.941468:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575537.941469:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575537.941503:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575537.941506:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575537.941508:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575538.069493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575538.069499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575538.069501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575538.069504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575538.069507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575538.069511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575538.069512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575538.069515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575538.069517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575538.965486:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575538.965491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575538.965495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575538.965496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575538.965498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575538.965498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575538.965554:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575538.965559:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575538.965562:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575539.093495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575539.093503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575539.093506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575539.093512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575539.093515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575539.093519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575539.093522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575539.093524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575539.093527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575539.605534:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575539.605545:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575539.605547:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575539.605552:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575539.605556:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575539.605564:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12470
00000400:00000200:0.0:1788575539.605574:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575539.605580:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575539.605583:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575539.605587:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605591:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575539.605594:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605596:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575539.605600:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575539.605602:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575539.605605:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575539.605608:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575539.605611:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575539.605613:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575539.605616:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605618:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575539.605620:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605622:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575539.605625:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575539.605628:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605631:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575539.605634:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575539.605637:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575539.605641:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575539.605643:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575539.605645:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575539.605654:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788575539.605658:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575539.605661:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575539.605663:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575539.605666:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575539.605668:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575539.605671:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575539.605679:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575539.605685:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575539.605708:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575539.605718:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575539.605722:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575539.605727:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575539.605747:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575539.605751:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575539.605775:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575539.605777:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575539.605780:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575539.605784:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575539.605789:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12470
00000400:00000200:0.0:1788575539.605794:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575539.605798:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1788575539.605800:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575539.605807:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575539.605807:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575539.605813:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575539.605820:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605827:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575539.605829:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575539.605834:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605843:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575539.605849:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575539.605852:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575539.605854:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1788575539.605857:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575539.605859:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575539.605862:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
02000000:00000001:3.0:1788575539.605866:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:0.0:1788575539.605867:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:3.0:1788575539.605872:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1788575539.605873:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:3.0:1788575539.605880:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575539.605882:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605887:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1788575539.605889:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393217024:192.168.202.110@tcp:250:kworker.0
00000100:00000040:0.0:1788575539.605895:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605899:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575539.605903:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575539.605911:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:0.0:1788575539.605912:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575539.605917:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575539.605920:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575539.605921:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575539.605929:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575539.605930:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575539.605936:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1788575539.605937:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575539.605941:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575539.605945:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:3.0:1788575539.605947:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
00000008:00000001:0.0:1788575539.605949:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575539.605952:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575539.605960:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575539.605963:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1788575539.605968:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000400:00000010:3.0:1788575539.605970:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000020:00000040:0.0:1788575539.605974:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575539.605979:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575539.605980:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575539.605981:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:0.0:1788575539.605983:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575539.605987:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575539.605989:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575539.605992:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393217024, portal 25
02000000:00000010:0.0:1788575539.605994:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:3.0:1788575539.605997:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575539.606000:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000001:0.0:1788575539.606002:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575539.606010:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575539.606013:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575539.606044:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575539.606050:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575539.606056:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575539.606065:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393217024, offset 0
00000100:00000001:0.0:1788575539.606071:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575539.606073:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575539.606077:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575539.606085:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575539.606087:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575539.606093:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575539.606103:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575539.606112:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575539.606121:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788575539.606122:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575539.606125:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:0.0:1788575539.606130:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575539.606131:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330951:330951:256:4294967295] 192.168.202.110@tcp LPNI seq info [330908:330908:8:4294967295]
00000400:00000200:3.0:1788575539.606143:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788575539.606145:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575539.606149:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575539.606153:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575539.606158:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788575539.606165:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:3.0:1788575539.606169:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788575539.606174:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12470
00000800:00000010:3.0:1788575539.606177:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000400:00000200:0.0:1788575539.606187:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:3.0:1788575539.606188:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575539.606195:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000400:00000200:0.0:1788575539.606199:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575539.606207:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:3.1:1788575539.606208:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788575539.606214:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575539.606216:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:0.0:1788575539.606221:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575539.606229:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000100:3.0:1788575539.606230:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575539.606236:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000010:3.0:1788575539.606237:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000020:00000001:0.0:1788575539.606244:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575539.606246:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:3.0:1788575539.606247:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1788575539.606249:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575539.606253:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575539.606258:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1788575539.606259:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575539.606266:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575539.606269:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000100:00000040:0.0:1788575539.606271:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:3.0:1788575539.606275:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:0.0:1788575539.606277:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575539.606283:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575539.606284:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:0.0:1788575539.606290:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:3.0:1788575539.606294:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575539.606296:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575539.606308:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575539.606316:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575539.606318:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575539.606321:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575539.606326:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575539.606329:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575539.606329:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000008:00100000:0.0:1788575539.606335:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000040:3.0:1788575539.606338:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1788575539.606340:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575539.606348:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575539.606357:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000100:00000001:3.0:1788575539.606360:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575539.606362:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575539.606365:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575539.606366:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575539.606373:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575539.606374:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575539.606378:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575539.606381:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575539.606384:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575539.606387:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000400:00000200:3.0:1788575539.606415:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575539.606418:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307144122
00000100:00000010:3.0:1788575539.606424:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:0.0:1788575539.606429:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575539.606433:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575539.606440:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307144122
00000100:00100000:0.0:1788575539.606441:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:3.0:1788575539.606444:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575539.606449:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575539.606454:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575539.606461:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575539.606462:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575539.606466:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1788575539.606467:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575539.606477:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575539/real 1788575539]  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575539.606479:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575539.606513:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1788575539.606515:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1788575539.606517:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1788575539.606521:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1788575539.606523:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575539.606528:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000100:00000040:0.0:1788575539.606530:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:3.0:1788575539.606532:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000040:0.0:1788575539.606539:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12480)
00000100:00000001:3.0:1788575539.606539:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575539.606548:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575539.606559:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606564:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575539.606565:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.606568:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575539.606569:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575539.606572:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606576:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575539.606580:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606587:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575539.606590:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.606592:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575539.606593:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.606596:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575539.606599:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575539.606602:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575539.606607:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575539.606610:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575539.606614:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575539.606623:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393217024:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575539.606630:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.606633:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575539.606636:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393217024/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606644:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575539.606647:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575539.606650:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575539.606653:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575539.606656:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575539.606658:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575539.606661:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575539.606664:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:3.0:1788575539.606668:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575539.606670:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575539.606672:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:3.0:1788575539.606675:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575539.606676:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.606679:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.606687:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575539.606690:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575539.606692:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575539.606709:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575539.606711:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575539.606716:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606724:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575539.606726:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575539.606728:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575539.606731:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575539.606733:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575539.606736:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217152:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575539.606739:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575539.606742:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575539.606743:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575539.606745:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575539.606749:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575539.606751:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575539.606755:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575539.606757:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575539.606761:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575539.606764:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393217152, portal 10
00000100:00000001:3.0:1788575539.606766:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575539.606768:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00000040:3.0:1788575539.606775:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606782:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575539.606784:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575539.606787:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575539.606789:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393217152, offset 0
00000400:00000200:3.0:1788575539.606793:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575539.606797:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575539.606802:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575539.606805:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575539.606807:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575539.606810:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575539.606812:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575539.606815:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330952:330952:256:4294967295] 192.168.202.110@tcp LPNI seq info [330909:330909:8:4294967295]
00000400:00000200:3.0:1788575539.606819:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575539.606825:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575539.606829:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575539.606832:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575539.606836:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575539.606838:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1788575539.606842:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575539.606845:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1788575539.606849:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575539.606851:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575539.606854:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575539.606857:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575539.606861:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575539.606862:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575539.606865:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575539.606868:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606876:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575539.606879:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606886:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575539.606888:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575539.606891:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575539.606895:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575539.606899:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575539.606901:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307143207
00000100:00000001:3.0:1788575539.606903:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307144122
00000100:00000001:3.0:1788575539.606906:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307143207
00000100:00000001:3.0:1788575539.606908:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307144122
00000100:00000001:3.0:1788575539.606911:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575539.606913:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575539.606915:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575539.606918:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575539/real 1788575539]  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575539.606926:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575539.606927:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575539.606929:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575539.606932:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575539.606933:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575539.606935:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575539.606940:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575539.606948:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606954:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575539.606955:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.606957:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575539.606959:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575539.606962:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606969:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575539.606973:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.606980:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575539.606983:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.606985:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575539.606986:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.606988:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575539.606991:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575539.606993:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575539.606997:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575539.607000:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575539.607004:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575539.607012:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217152:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575539.607016:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575539.607020:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.607033:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575539.607035:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575539.607037:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575539.607039:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575539.607041:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575539.607044:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217280:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575539.607047:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575539.607049:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575539.607051:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575539.607053:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575539.607056:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575539.607058:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575539.607062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575539.607065:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575539.607068:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575539.607071:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393217280, portal 4
00000100:00000001:3.0:1788575539.607073:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575539.607074:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:3.0:1788575539.607080:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.607087:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575539.607089:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575539.607091:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575539.607094:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393217280, offset 0
00000400:00000200:3.0:1788575539.607098:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575539.607102:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575539.607106:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575539.607109:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575539.607111:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575539.607113:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575539.607116:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575539.607119:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330953:330953:256:4294967295] 192.168.202.110@tcp LPNI seq info [330910:330910:8:4294967295]
00000400:00000200:3.0:1788575539.607124:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575539.607130:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575539.607135:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575539.607138:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575539.607142:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575539.607145:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1788575539.607149:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575539.607152:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1788575539.607156:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575539.607158:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575539.607161:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575539.607164:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575539.607167:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575539.607169:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575539.607171:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575539.607175:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.607182:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575539.607185:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.607192:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575539.607194:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575539.607197:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575539.607199:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575539.607202:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575539.607204:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307143353
00000100:00000001:3.0:1788575539.607207:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307144122
00000100:00000001:3.0:1788575539.607209:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307143353
00000100:00000001:3.0:1788575539.607211:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307144122
00000100:00000001:3.0:1788575539.607214:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575539.607216:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575539.607218:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575539.607222:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575539/real 1788575539]  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575539.607230:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575539.607232:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575539.607234:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575539.607236:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575539.607238:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575539.607240:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575539.607243:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575539.607252:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.607258:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575539.607260:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.607261:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575539.607262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575539.607265:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.607271:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575539.607275:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.607281:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575539.607283:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.607285:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575539.607286:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.607288:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575539.607290:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575539.607292:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575539.607296:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575539.607299:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575539.607303:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575539.607310:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217280:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575539.607314:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.607316:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575539.607320:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393217152/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.607327:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575539.607328:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575539.607331:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575539.607333:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575539.607335:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575539.607336:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575539.607339:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575539.607340:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:3.0:1788575539.607343:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575539.607344:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575539.607346:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:3.0:1788575539.607348:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575539.607349:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.607351:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575539.607354:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393217280/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575539.607360:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575539.607362:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575539.607363:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575539.607366:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575539.607367:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575539.607368:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575539.607370:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575539.607373:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:3.0:1788575539.607375:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575539.607376:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575539.607377:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:3.0:1788575539.607380:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575539.607381:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.607383:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575539.607387:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575539.607388:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575539.607410:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575539.607412:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575539.989557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575539.989565:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575539.989572:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575539.989573:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575539.989577:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575539.989578:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575539.989657:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575539.989661:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575539.989665:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575540.117523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575540.117531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575540.117532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575540.117538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575540.117541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575540.117545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575540.117547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575540.117549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575540.117551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575540.565535:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307144122, at 4307144128
00000100:00000040:1.0:1788575540.565535:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307144122, at 4307144128
00000100:00000010:0.0:1788575540.565555:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788575540.565557:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575541.013531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575541.013540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575541.013546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575541.013547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575541.013552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575541.013553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575541.013618:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575541.013621:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575541.013624:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575541.141521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575541.141528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575541.141531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575541.141536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575541.141539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575541.141543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575541.141545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575541.141547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575541.141550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575542.037552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575542.037561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575542.037568:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575542.037569:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575542.037573:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575542.037575:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575542.037647:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575542.037650:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575542.037653:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575542.165516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575542.165526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575542.165529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575542.165535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575542.165539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575542.165543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575542.165546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575542.165548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575542.165551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575543.061503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575543.061510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575543.061515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575543.061516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575543.061518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575543.061519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575543.061592:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575543.061599:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575543.061604:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575543.189525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575543.189532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575543.189534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575543.189543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575543.189547:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575543.189551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575543.189553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575543.189556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575543.189559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575544.085499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575544.085504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575544.085509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575544.085511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575544.085514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575544.085515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575544.085578:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575544.085582:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575544.085585:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575544.213499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575544.213505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575544.213507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575544.213512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575544.213515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575544.213518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575544.213521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575544.213524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575544.213527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575544.725519:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575544.725527:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575544.725529:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575544.725533:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575544.725536:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575544.725542:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12475
00000400:00000200:2.0:1788575544.725548:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575544.725553:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575544.725555:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575544.725557:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725559:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575544.725561:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725562:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575544.725565:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575544.725566:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575544.725568:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575544.725570:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575544.725571:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575544.725573:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575544.725574:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725575:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575544.725577:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725578:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575544.725580:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575544.725582:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725584:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575544.725585:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575544.725588:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575544.725590:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575544.725591:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575544.725593:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575544.725601:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01c00.
00000020:00000040:2.0:1788575544.725604:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575544.725607:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575544.725609:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575544.725611:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575544.725613:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575544.725616:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575544.725623:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11400.
00000100:00000001:2.0:1788575544.725630:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575544.725640:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575544.725649:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575544.725654:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575544.725658:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000de3acaa1] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575544.725673:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575544.725675:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575544.725677:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575544.725678:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575544.725680:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575544.725681:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575544.725683:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12475
00000400:00000200:2.0:1788575544.725686:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575544.725688:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575544.725689:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575544.725691:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725693:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575544.725694:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725696:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575544.725697:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575544.725698:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575544.725699:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575544.725701:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575544.725702:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575544.725703:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575544.725708:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725709:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575544.725711:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725713:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575544.725714:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575544.725716:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725718:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575544.725719:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575544.725722:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1788575544.725723:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575544.725726:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575544.725728:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575544.725728:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000008:00100000:2.0:1788575544.725733:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575544.725733:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000008:00000001:2.0:1788575544.725736:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575544.725738:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575544.725741:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00e00.
00000100:00000040:3.0:1788575544.725742:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788575544.725743:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575544.725746:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575544.725748:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575544.725751:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575544.725753:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575544.725755:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575544.725759:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12c00.
00000100:00000001:3.0:1788575544.725761:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575544.725763:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.725765:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575544.725769:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1788575544.725771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575544.725788:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575544.725791:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575544.725796:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393217408:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575544.725798:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575544.725801:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.725804:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:2.0:1788575544.725806:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5f665f0] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575544.725810:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575544.725814:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575544.725815:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575544.725820:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:3.0:1788575544.725820:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:2.0:1788575544.725824:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575544.725825:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575544.725827:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575544.725828:0:186668:0:(import.c:520:import_select_connection()) Process entered
02000000:00000010:3.0:1788575544.725828:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
00000100:00080000:2.0:1788575544.725832:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12475
02000000:00000001:3.0:1788575544.725833:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575544.725836:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575544.725839:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575544.725840:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575544.725844:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575544.725844:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575544.725848:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575544.725850:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725851:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575544.725852:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000010:3.0:1788575544.725852:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000020:00000040:2.0:1788575544.725854:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575544.725857:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575544.725858:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575544.725860:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575544.725861:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:3.0:1788575544.725862:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000040:2.0:1788575544.725863:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725865:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575544.725866:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725867:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575544.725869:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575544.725870:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575544.725872:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575544.725873:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575544.725875:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575544.725877:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575544.725878:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575544.725880:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000200:3.0:1788575544.725880:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393217408, portal 25
00000008:00000001:2.0:1788575544.725886:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575544.725887:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.725887:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575544.725891:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000010:2.0:1788575544.725892:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03100.
00000020:00000040:2.0:1788575544.725896:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575544.725898:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575544.725900:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575544.725901:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575544.725902:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575544.725904:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575544.725906:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575544.725911:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000100:00000001:3.0:1788575544.725914:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1788575544.725915:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575544.725918:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00100000:2.0:1788575544.725923:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575544.725923:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575544.725933:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393217408, offset 0
00000100:00000001:2.0:1788575544.725940:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575544.725944:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575544.725945:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575544.725950:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008692924c] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575544.725956:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575544.725957:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575544.725960:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575544.725963:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575544.725966:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12485)
00000400:00000200:3.0:1788575544.725966:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575544.725975:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575544.725979:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575544.725983:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575544.725986:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575544.725989:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330954:330954:256:4294967295] 192.168.202.110@tcp LPNI seq info [330911:330911:8:4294967295]
00000400:00000200:3.0:1788575544.725994:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575544.726004:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575544.726010:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575544.726014:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575544.726019:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575544.726021:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1788575544.726025:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575544.726027:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1788575544.726030:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575544.726031:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575544.726037:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575544.726040:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575544.726044:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575544.726046:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575544.726049:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575544.726052:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726060:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575544.726064:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726079:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.726081:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575544.726088:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575544.726092:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575544.726095:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307149241
00000100:00000010:3.0:1788575544.726101:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575544.726107:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575544.726116:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307149241
00000100:00000010:3.0:1788575544.726118:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788575544.726121:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575544.726123:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575544.726126:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.726128:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575544.726134:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575544/real 1788575544]  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575544.726143:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575544.726145:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575544.726147:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575544.726150:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575544.726151:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575544.726153:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575544.726157:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575544.726166:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726172:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575544.726174:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726177:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575544.726179:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575544.726182:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726189:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575544.726193:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726200:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575544.726202:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726204:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575544.726204:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726206:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575544.726208:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575544.726210:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575544.726212:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575544.726213:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575544.726216:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575544.726220:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393217408:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575544.726224:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726225:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575544.726227:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393217408/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726230:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575544.726232:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575544.726234:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575544.726236:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575544.726237:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575544.726238:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575544.726240:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575544.726241:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575544.726244:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575544.726245:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575544.726246:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:3.0:1788575544.726248:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575544.726249:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726250:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726256:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575544.726257:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.726258:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575544.726259:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575544.726260:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575544.726263:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726267:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575544.726275:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575544.726277:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575544.726278:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575544.726281:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575544.726283:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217536:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575544.726286:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575544.726288:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575544.726290:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575544.726292:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575544.726294:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575544.726296:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575544.726300:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575544.726303:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575544.726306:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575544.726318:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393217536, portal 10
00000100:00000001:3.0:1788575544.726320:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575544.726325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:3.0:1788575544.726331:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726340:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575544.726342:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575544.726345:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575544.726348:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393217536, offset 0
00000400:00000200:3.0:1788575544.726352:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575544.726356:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575544.726360:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575544.726363:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575544.726366:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575544.726368:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575544.726370:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575544.726373:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330955:330955:256:4294967295] 192.168.202.110@tcp LPNI seq info [330912:330912:8:4294967295]
00000400:00000200:3.0:1788575544.726377:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575544.726383:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575544.726388:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575544.726413:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575544.726419:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575544.726421:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1788575544.726426:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575544.726428:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1788575544.726432:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575544.726434:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575544.726437:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575544.726440:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575544.726443:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575544.726445:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575544.726448:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575544.726452:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726460:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575544.726463:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726469:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.726471:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575544.726474:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575544.726477:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575544.726481:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575544.726483:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307148327
00000100:00000001:3.0:1788575544.726486:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307149241
00000100:00000001:3.0:1788575544.726488:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307148327
00000100:00000001:3.0:1788575544.726490:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307149241
00000100:00000001:3.0:1788575544.726492:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575544.726494:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.726496:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575544.726500:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575544/real 1788575544]  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575544.726508:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575544.726509:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575544.726510:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575544.726513:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575544.726514:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575544.726516:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575544.726520:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575544.726525:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726529:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575544.726530:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726531:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575544.726532:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575544.726533:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726537:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575544.726539:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726544:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575544.726545:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726546:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575544.726547:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726548:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575544.726549:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575544.726558:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575544.726560:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575544.726562:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575544.726564:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575544.726569:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217536:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575544.726571:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575544.726572:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726576:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575544.726577:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575544.726578:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575544.726580:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575544.726581:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575544.726582:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575544.726584:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575544.726585:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575544.726586:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575544.726587:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575544.726589:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575544.726591:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575544.726593:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575544.726595:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575544.726597:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575544.726599:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393217664, portal 4
00000100:00000001:3.0:1788575544.726600:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575544.726601:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:3.0:1788575544.726604:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726611:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575544.726613:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575544.726614:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575544.726616:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393217664, offset 0
00000400:00000200:3.0:1788575544.726618:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575544.726620:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575544.726622:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575544.726624:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575544.726626:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575544.726627:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575544.726629:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575544.726631:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330956:330956:256:4294967295] 192.168.202.110@tcp LPNI seq info [330913:330913:8:4294967295]
00000400:00000200:3.0:1788575544.726633:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575544.726638:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575544.726640:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575544.726642:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575544.726645:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575544.726646:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1788575544.726649:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575544.726650:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1788575544.726653:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575544.726654:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575544.726656:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575544.726658:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575544.726660:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575544.726661:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575544.726662:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575544.726664:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726669:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575544.726670:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726678:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.726679:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575544.726681:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575544.726683:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575544.726685:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575544.726686:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307148473
00000100:00000001:3.0:1788575544.726688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307149241
00000100:00000001:3.0:1788575544.726689:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307148473
00000100:00000001:3.0:1788575544.726690:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307149241
00000100:00000001:3.0:1788575544.726691:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575544.726692:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.726693:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575544.726695:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575544/real 1788575544]  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575544.726699:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575544.726700:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575544.726701:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575544.726702:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575544.726703:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575544.726704:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575544.726706:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575544.726710:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726714:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575544.726714:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726715:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575544.726716:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575544.726718:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726723:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575544.726725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726728:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575544.726730:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575544.726731:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726732:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575544.726733:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575544.726734:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575544.726736:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575544.726737:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575544.726739:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575544.726743:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575544.726745:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726746:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575544.726753:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393217536/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726756:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575544.726757:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575544.726759:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575544.726760:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575544.726761:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575544.726762:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575544.726763:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575544.726764:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575544.726766:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575544.726767:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575544.726768:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:3.0:1788575544.726770:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575544.726771:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726772:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575544.726788:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393217664/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575544.726793:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575544.726794:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575544.726795:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575544.726797:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575544.726798:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575544.726799:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575544.726800:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575544.726801:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575544.726802:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575544.726802:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575544.726803:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:3.0:1788575544.726804:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575544.726805:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726806:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575544.726808:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575544.726809:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575544.726811:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575544.726812:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575545.109484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575545.109489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575545.109492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575545.109494:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575545.109496:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575545.109498:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575545.109501:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575545.109502:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575545.109504:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575545.109505:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575545.109507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575545.109508:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575545.109509:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575545.109510:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575545.109511:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575545.109512:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575545.109513:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575545.109514:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575545.109574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575545.109578:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575545.109581:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575545.237494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575545.237500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575545.237501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575545.237505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575545.237508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575545.237511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575545.237513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575545.237515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575545.237517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575545.685516:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307149241, at 4307149248
00000100:00000040:1.0:1788575545.685517:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307149241, at 4307149248
00000100:00000010:0.0:1788575545.685538:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788575545.685540:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00010000:00000001:3.0:1788575546.133529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575546.133536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575546.133546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575546.133547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575546.133549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575546.133550:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575546.133612:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575546.133616:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575546.133618:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575546.261497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575546.261506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575546.261509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575546.261515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575546.261518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575546.261523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575546.261525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575546.261527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575546.261530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575547.157464:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575547.157471:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575547.157474:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575547.157477:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575547.157481:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575547.157483:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575547.157485:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575547.157488:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575547.157491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575547.157493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575547.157496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575547.157497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575547.157568:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575547.157573:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575547.157576:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575547.285500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575547.285506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575547.285509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575547.285513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575547.285517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575547.285521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575547.285523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575547.285526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575547.285529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575548.181505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575548.181511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575548.181514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575548.181515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575548.181517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575548.181518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575548.181574:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575548.181577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575548.181580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575548.309462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575548.309469:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575548.309471:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575548.309476:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575548.309479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575548.309483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575548.309485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575548.309487:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575548.309498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575549.205552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575549.205561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575549.205567:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575549.205568:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575549.205572:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575549.205573:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575549.205665:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575549.205672:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575549.205676:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575549.333551:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575549.333559:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575549.333561:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575549.333566:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575549.333569:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575549.333572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575549.333575:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575549.333577:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575549.333579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575549.845507:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575549.845518:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575549.845520:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575549.845524:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575549.845530:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575549.845536:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12480
00000400:00000200:0.0:1788575549.845553:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575549.845559:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575549.845562:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575549.845565:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575549.845568:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575549.845571:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575549.845574:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575549.845578:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575549.845579:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575549.845582:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575549.845585:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575549.845588:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575549.845590:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575549.845593:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575549.845595:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575549.845597:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575549.845599:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575549.845602:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575549.845612:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575549.845615:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575549.845618:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575549.845621:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575549.845624:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575549.845627:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575549.845629:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575549.845638:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788575549.845642:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575549.845645:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575549.845647:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575549.845650:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575549.845653:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575549.845657:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575549.845664:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575549.845670:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575549.845679:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575549.845689:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575549.845693:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575549.845697:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575549.845719:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575549.845723:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575549.845726:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575549.845728:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575549.845730:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575549.845733:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575549.845737:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12480
00000400:00000200:0.0:1788575549.845744:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575549.845748:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575549.845751:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575549.845754:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1788575549.845755:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575549.845761:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575549.845762:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575549.845766:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575549.845769:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575549.845775:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575549.845780:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:3.0:1788575549.845780:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575549.845784:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575549.845791:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575549.845793:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575549.845798:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575549.845808:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575549.845809:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575549.845812:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575549.845817:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1788575549.845818:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575549.845822:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575549.845823:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1788575549.845828:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575549.845833:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00100000:3.0:1788575549.845837:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393217792:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575549.845839:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575549.845845:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1788575549.845846:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575549.845851:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575549.845866:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575549.845870:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575549.845870:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788575549.845876:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575549.845879:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:3.0:1788575549.845882:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:0.0:1788575549.845886:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575549.845886:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575549.845895:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575549.845898:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575549.845903:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:0.0:1788575549.845904:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575549.845911:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575549.845911:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:0.0:1788575549.845917:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575549.845923:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000010:0.0:1788575549.845924:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788575549.845929:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000200:3.0:1788575549.845933:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393217792, portal 25
02000000:00000001:0.0:1788575549.845934:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575549.845939:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.845939:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575549.845941:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523511296 : -130484186040320 : ffff895348713800)
00000100:00000001:0.0:1788575549.845948:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575549.845953:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1788575549.845954:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575549.845960:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.845966:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000010:0.0:1788575549.845970:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000040:3.0:1788575549.845973:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575549.845980:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575549.845981:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575549.845990:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393217792, offset 0
00000100:00100000:0.0:1788575549.845994:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575549.846004:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575549.846017:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575549.846029:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575549.846032:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788575549.846033:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575549.846041:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1788575549.846043:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575549.846051:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575549.846061:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788575549.846063:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575549.846064:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:0.0:1788575549.846071:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575549.846072:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330957:330957:256:4294967295] 192.168.202.110@tcp LPNI seq info [330914:330914:8:4294967295]
00000100:00080000:0.0:1788575549.846088:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:3.0:1788575549.846089:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575549.846096:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575549.846098:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575549.846107:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:3.0:1788575549.846108:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1788575549.846115:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12480
00000800:00000200:3.0:1788575549.846119:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575549.846126:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000400:00000200:0.0:1788575549.846127:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575549.846137:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:3.0:1788575549.846137:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575549.846144:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00080000:0.0:1788575549.846153:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:3.1:1788575549.846155:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575549.846160:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:0.0:1788575549.846163:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575549.846169:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000100:3.0:1788575549.846170:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788575549.846179:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:3.0:1788575549.846179:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000100:00000001:0.0:1788575549.846188:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575549.846191:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1788575549.846193:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:0.0:1788575549.846194:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575549.846202:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575549.846206:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575549.846208:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1788575549.846214:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575549.846218:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000020:00000001:0.0:1788575549.846219:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:3.0:1788575549.846222:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000040:0.0:1788575549.846229:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575549.846231:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:0.0:1788575549.846233:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575549.846243:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:3.0:1788575549.846244:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575549.846248:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575549.846259:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575549.846267:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575549.846271:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.846271:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1788575549.846278:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1788575549.846280:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575549.846282:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575549.846292:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575549.846299:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575549.846303:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1788575549.846304:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:3.0:1788575549.846309:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000008:00000001:0.0:1788575549.846312:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575549.846316:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.846319:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575549.846322:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788575549.846327:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575549.846328:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575549.846336:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1788575549.846336:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307154361
02000000:00000001:0.0:1788575549.846342:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575549.846345:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:0.0:1788575549.846346:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575549.846354:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1788575549.846358:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:0.0:1788575549.846359:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.846365:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307154361
02000000:00000010:0.0:1788575549.846368:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000010:3.0:1788575549.846370:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:0.0:1788575549.846374:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.846375:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575549.846381:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575549.846384:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.846389:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575549.846390:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1788575549.846431:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575549/real 1788575549]  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575549.846432:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575549.846437:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575549.846441:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575549.846512:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1788575549.846514:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1788575549.846516:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:3.0:1788575549.846516:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1788575549.846523:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:3.0:1788575549.846523:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575549.846530:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000100:00000040:0.0:1788575549.846533:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12491)
00000400:00000010:3.0:1788575549.846533:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575549.846542:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575549.846548:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575549.846556:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846561:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575549.846562:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.846564:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575549.846565:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575549.846567:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846572:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575549.846575:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846580:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575549.846582:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.846583:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575549.846584:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.846585:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575549.846587:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575549.846590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575549.846597:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575549.846601:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575549.846604:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575549.846609:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393217792:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575549.846613:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.846615:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575549.846617:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713800 x1875444393217792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846620:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575549.846622:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575549.846625:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575549.846628:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575549.846630:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575549.846631:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575549.846633:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575549.846635:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:3.0:1788575549.846639:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575549.846640:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575549.846642:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713800.
00000100:00000001:3.0:1788575549.846644:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575549.846645:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.846647:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.846653:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575549.846655:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.846656:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575549.846658:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575549.846659:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575549.846661:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846670:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575549.846671:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575549.846672:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575549.846674:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575549.846676:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575549.846677:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575549.846679:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575549.846681:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575549.846682:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575549.846683:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575549.846685:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575549.846686:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575549.846688:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575549.846690:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575549.846693:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575549.846695:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393217920, portal 10
00000100:00000001:3.0:1788575549.846696:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575549.846697:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523503232 : -130484186048384 : ffff895348711880)
00000100:00000040:3.0:1788575549.846702:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846706:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575549.846708:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575549.846709:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575549.846711:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393217920, offset 0
00000400:00000200:3.0:1788575549.846713:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575549.846715:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575549.846718:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575549.846719:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575549.846721:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575549.846723:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575549.846724:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575549.846727:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330958:330958:256:4294967295] 192.168.202.110@tcp LPNI seq info [330915:330915:8:4294967295]
00000400:00000200:3.0:1788575549.846729:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575549.846734:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575549.846736:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575549.846739:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575549.846741:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575549.846743:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788575549.846747:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575549.846748:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1788575549.846751:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575549.846752:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575549.846754:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575549.846756:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575549.846758:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575549.846759:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575549.846761:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575549.846763:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846768:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575549.846769:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846773:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.846774:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575549.846776:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575549.846779:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575549.846780:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575549.846782:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307153447
00000100:00000001:3.0:1788575549.846783:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307154361
00000100:00000001:3.0:1788575549.846785:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307153447
00000100:00000001:3.0:1788575549.846786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307154361
00000100:00000001:3.0:1788575549.846788:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575549.846789:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.846791:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575549.846793:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575549/real 1788575549]  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575549.846798:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575549.846798:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575549.846799:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575549.846800:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575549.846801:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575549.846802:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575549.846804:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575549.846809:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846813:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575549.846813:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.846814:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575549.846815:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575549.846817:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846820:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575549.846822:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846826:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575549.846827:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.846828:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575549.846829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.846829:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575549.846831:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575549.846832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575549.846835:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575549.846836:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575549.846838:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575549.846843:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393217920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575549.846845:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575549.846847:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846850:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575549.846851:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575549.846852:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575549.846853:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575549.846867:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575549.846870:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218048:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575549.846873:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575549.846875:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575549.846876:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575549.846878:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575549.846881:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575549.846883:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575549.846887:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575549.846889:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575549.846893:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575549.846896:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393218048, portal 4
00000100:00000001:3.0:1788575549.846898:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575549.846899:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:3.0:1788575549.846906:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846911:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575549.846913:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575549.846915:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575549.846917:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393218048, offset 0
00000400:00000200:3.0:1788575549.846919:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575549.846922:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575549.846924:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575549.846926:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575549.846928:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575549.846929:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575549.846931:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575549.846933:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330959:330959:256:4294967295] 192.168.202.110@tcp LPNI seq info [330916:330916:8:4294967295]
00000400:00000200:3.0:1788575549.846936:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575549.846940:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575549.846942:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575549.846944:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575549.846947:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575549.846949:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788575549.846951:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575549.846953:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1788575549.846955:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575549.846957:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575549.846959:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575549.846961:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575549.846963:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575549.846964:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575549.846966:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575549.846969:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846977:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575549.846980:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.846986:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.846987:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575549.846991:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575549.846994:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575549.846997:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575549.846999:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307153593
00000100:00000001:3.0:1788575549.847002:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307154361
00000100:00000001:3.0:1788575549.847004:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307153593
00000100:00000001:3.0:1788575549.847006:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307154361
00000100:00000001:3.0:1788575549.847008:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575549.847010:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.847012:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575549.847015:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575549/real 1788575549]  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575549.847023:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575549.847024:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575549.847026:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575549.847029:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575549.847030:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575549.847032:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575549.847036:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575549.847044:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.847051:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575549.847052:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.847054:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575549.847055:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575549.847058:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.847062:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575549.847064:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.847069:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575549.847071:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.847071:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575549.847072:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.847073:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575549.847074:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575549.847076:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575549.847078:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575549.847079:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575549.847081:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575549.847085:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218048:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575549.847088:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.847089:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575549.847091:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711880 x1875444393217920/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.847095:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575549.847096:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575549.847098:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575549.847099:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575549.847100:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575549.847101:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575549.847102:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575549.847104:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:3.0:1788575549.847105:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575549.847106:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575549.847107:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711880.
00000100:00000001:3.0:1788575549.847109:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575549.847110:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.847111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575549.847113:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393218048/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575549.847117:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575549.847118:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575549.847118:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575549.847120:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575549.847120:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575549.847121:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575549.847122:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575549.847123:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:3.0:1788575549.847124:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575549.847124:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575549.847125:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:3.0:1788575549.847126:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575549.847127:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.847128:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575549.847130:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575549.847131:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575549.847132:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575549.847133:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575550.229546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575550.229555:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575550.229562:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575550.229563:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575550.229566:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575550.229567:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575550.229610:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575550.229617:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575550.229622:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575550.357550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575550.357560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575550.357563:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575550.357570:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575550.357574:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575550.357580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575550.357583:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575550.357587:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575550.357589:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575550.805514:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307154361, at 4307154368
00000100:00000040:1.0:1788575550.805515:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307154361, at 4307154368
00000100:00000010:0.0:1788575550.805532:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:1.0:1788575550.805534:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575551.253529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575551.253538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575551.253546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575551.253547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575551.253551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575551.253553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575551.253621:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575551.253626:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575551.253629:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575551.381539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575551.381549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575551.381552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575551.381559:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575551.381563:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575551.381567:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575551.381570:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575551.381573:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575551.381576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575552.277560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575552.277570:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575552.277578:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575552.277579:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575552.277583:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575552.277585:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575552.277682:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575552.277688:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575552.277693:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575552.405535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575552.405546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575552.405549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575552.405556:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575552.405560:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575552.405565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575552.405567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575552.405570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575552.405574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575553.301545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575553.301552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575553.301557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575553.301558:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575553.301561:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575553.301562:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575553.301648:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575553.301654:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575553.301659:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575553.429544:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575553.429553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575553.429556:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575553.429563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575553.429566:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575553.429572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575553.429575:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575553.429578:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575553.429581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575554.325562:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575554.325571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575554.325577:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575554.325578:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575554.325582:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575554.325584:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575554.325660:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575554.325664:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575554.325675:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575554.453508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575554.453516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575554.453518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575554.453523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575554.453526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575554.453529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575554.453532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575554.453534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575554.453536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575554.965491:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575554.965500:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575554.965503:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575554.965507:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575554.965511:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575554.965520:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12486
00000400:00000200:0.0:1788575554.965532:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575554.965540:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575554.965543:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575554.965548:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965552:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575554.965562:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965565:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575554.965570:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575554.965572:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575554.965574:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575554.965578:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575554.965579:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575554.965581:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575554.965583:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965584:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575554.965585:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965586:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575554.965589:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575554.965591:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965593:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575554.965596:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575554.965599:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575554.965601:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575554.965603:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575554.965606:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575554.965616:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788575554.965624:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575554.965628:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575554.965629:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575554.965632:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575554.965634:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575554.965639:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575554.965646:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788575554.965652:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575554.965665:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575554.965673:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575554.965678:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575554.965684:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575554.965704:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575554.965708:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575554.965711:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575554.965713:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575554.965718:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575554.965722:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575554.965724:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12486
00000400:00000200:0.0:1788575554.965728:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575554.965730:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1788575554.965731:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1788575554.965732:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575554.965737:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965738:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575554.965740:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575554.965741:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575554.965742:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575554.965745:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575554.965747:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575554.965748:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575554.965750:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575554.965750:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1788575554.965752:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575554.965754:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575554.965757:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965758:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575554.965760:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965761:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575554.965763:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575554.965766:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965768:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575554.965771:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575554.965775:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575554.965783:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575554.965785:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788575554.965787:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575554.965790:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575554.965790:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575554.965796:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:0.0:1788575554.965797:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575554.965800:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:0.0:1788575554.965802:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575554.965807:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575554.965807:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575554.965815:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000100:00100000:3.0:1788575554.965816:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393218176:192.168.202.110@tcp:250:kworker.0
00000020:00000040:0.0:1788575554.965818:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575554.965823:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575554.965825:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575554.965826:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1788575554.965827:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575554.965830:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575554.965832:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575554.965835:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:3.0:1788575554.965835:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1788575554.965841:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575554.965842:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575554.965846:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1788575554.965850:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788575554.965854:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575554.965856:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575554.965859:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788575554.965861:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575554.965866:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575554.965871:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575554.965873:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575554.965878:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788575554.965879:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575554.965882:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575554.965884:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575554.965886:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:3.0:1788575554.965886:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00080000:0.0:1788575554.965890:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12486
00000400:00000200:0.0:1788575554.965894:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575554.965896:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575554.965898:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1788575554.965899:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000040:0.0:1788575554.965901:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965904:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575554.965905:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965907:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575554.965908:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575554.965909:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000200:3.0:1788575554.965909:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393218176, portal 25
00000020:00000001:0.0:1788575554.965912:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575554.965915:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:3.0:1788575554.965915:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000020:00000001:0.0:1788575554.965920:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:3.0:1788575554.965920:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000020:00000001:0.0:1788575554.965926:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575554.965932:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575554.965932:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575554.965934:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575554.965961:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575554.965964:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788575554.965965:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575554.965969:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575554.965972:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575554.965975:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575554.965978:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575554.965983:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1788575554.965983:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575554.965987:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575554.965991:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575554.965994:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000010:3.0:1788575554.965994:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000008:00000001:0.0:1788575554.965999:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575554.966002:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575554.966007:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000100:00000200:3.0:1788575554.966008:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393218176, offset 0
00000020:00000040:0.0:1788575554.966010:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575554.966015:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575554.966016:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575554.966019:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575554.966022:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575554.966022:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575554.966025:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575554.966033:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000400:00000200:3.0:1788575554.966034:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575554.966039:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575554.966045:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:0.0:1788575554.966047:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575554.966055:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575554.966059:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575554.966064:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575554.966064:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575554.966072:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575554.966075:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:0.0:1788575554.966079:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575554.966084:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330960:330960:256:4294967295] 192.168.202.110@tcp LPNI seq info [330917:330917:8:4294967295]
00000100:00000001:0.0:1788575554.966092:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575554.966095:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575554.966098:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1788575554.966102:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575554.966103:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12496)
00000400:00000200:3.0:1788575554.966127:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575554.966134:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575554.966140:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575554.966147:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575554.966151:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575554.966157:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575554.966161:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575554.966166:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575554.966169:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575554.966176:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575554.966181:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575554.966188:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575554.966190:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575554.966196:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575554.966200:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966209:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575554.966215:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966224:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575554.966226:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575554.966238:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575554.966244:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575554.966247:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307159481
00000100:00000010:3.0:1788575554.966253:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575554.966261:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575554.966275:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307159481
00000100:00000010:3.0:1788575554.966279:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575554.966283:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575554.966288:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575554.966291:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575554.966294:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575554.966299:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575554/real 1788575554]  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575554.966310:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575554.966312:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575554.966314:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575554.966317:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575554.966319:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575554.966322:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575554.966326:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575554.966333:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966339:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575554.966341:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.966344:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575554.966346:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575554.966350:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966357:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575554.966362:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966369:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575554.966372:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.966374:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575554.966375:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.966377:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575554.966379:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575554.966383:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575554.966388:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575554.966429:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575554.966436:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575554.966446:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393218176:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575554.966451:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.966454:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575554.966457:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393218176/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966463:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575554.966466:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575554.966469:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575554.966472:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575554.966474:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575554.966476:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575554.966478:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575554.966481:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:3.0:1788575554.966484:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575554.966486:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575554.966488:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:3.0:1788575554.966491:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575554.966493:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.966495:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.966504:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575554.966506:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575554.966508:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575554.966509:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575554.966511:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575554.966515:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966522:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575554.966523:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575554.966525:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575554.966527:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575554.966529:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575554.966532:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218304:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575554.966535:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575554.966537:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575554.966539:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575554.966541:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575554.966544:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575554.966546:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575554.966549:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575554.966552:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575554.966556:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575554.966559:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393218304, portal 10
00000100:00000001:3.0:1788575554.966561:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575554.966563:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000040:3.0:1788575554.966570:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966576:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575554.966579:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575554.966582:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575554.966585:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393218304, offset 0
00000400:00000200:3.0:1788575554.966589:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575554.966593:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575554.966596:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575554.966599:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575554.966602:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575554.966604:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575554.966606:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575554.966609:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330961:330961:256:4294967295] 192.168.202.110@tcp LPNI seq info [330918:330918:8:4294967295]
00000400:00000200:3.0:1788575554.966613:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575554.966619:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575554.966624:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575554.966627:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575554.966630:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575554.966633:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575554.966637:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575554.966639:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575554.966643:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575554.966644:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575554.966647:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575554.966650:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575554.966653:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575554.966655:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575554.966658:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575554.966661:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966668:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575554.966671:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966677:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575554.966678:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575554.966682:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575554.966686:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575554.966689:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575554.966691:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307158567
00000100:00000001:3.0:1788575554.966693:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307159481
00000100:00000001:3.0:1788575554.966696:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307158567
00000100:00000001:3.0:1788575554.966698:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307159481
00000100:00000001:3.0:1788575554.966699:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575554.966702:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575554.966704:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575554.966707:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575554/real 1788575554]  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575554.966715:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575554.966717:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575554.966718:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575554.966721:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575554.966723:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575554.966725:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575554.966728:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575554.966737:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966742:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575554.966744:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.966746:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575554.966747:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575554.966750:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966759:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575554.966765:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966773:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575554.966776:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.966778:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575554.966779:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.966781:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575554.966783:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575554.966786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575554.966790:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575554.966792:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575554.966796:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575554.966803:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218304:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575554.966807:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575554.966810:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966816:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575554.966817:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575554.966819:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575554.966821:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575554.966822:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575554.966825:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218432:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575554.966828:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575554.966830:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575554.966831:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575554.966833:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575554.966836:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575554.966838:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575554.966841:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575554.966844:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575554.966847:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575554.966850:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393218432, portal 4
00000100:00000001:3.0:1788575554.966852:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575554.966853:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:3.0:1788575554.966860:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966868:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575554.966871:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575554.966874:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575554.966878:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393218432, offset 0
00000400:00000200:3.0:1788575554.966884:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575554.966889:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575554.966893:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575554.966897:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575554.966899:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575554.966901:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575554.966904:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575554.966907:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330962:330962:256:4294967295] 192.168.202.110@tcp LPNI seq info [330919:330919:8:4294967295]
00000400:00000200:3.0:1788575554.966911:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575554.966916:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575554.966921:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575554.966924:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575554.966928:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575554.966930:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575554.966948:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575554.966951:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575554.966955:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575554.966957:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575554.966960:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575554.966964:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575554.966967:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575554.966968:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575554.966971:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575554.966975:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966982:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575554.966985:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.966991:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575554.966992:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575554.966995:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575554.966999:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575554.967002:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575554.967004:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307158713
00000100:00000001:3.0:1788575554.967006:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307159481
00000100:00000001:3.0:1788575554.967008:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307158713
00000100:00000001:3.0:1788575554.967011:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307159481
00000100:00000001:3.0:1788575554.967013:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575554.967015:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575554.967017:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575554.967023:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575554/real 1788575554]  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575554.967033:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575554.967035:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575554.967036:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575554.967040:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575554.967042:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575554.967044:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575554.967048:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575554.967056:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.967067:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575554.967068:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.967070:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575554.967071:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575554.967075:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.967081:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575554.967084:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.967091:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575554.967093:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.967094:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575554.967096:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.967098:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575554.967099:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575554.967102:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575554.967104:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575554.967106:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575554.967110:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575554.967117:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218432:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575554.967121:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.967122:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575554.967125:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393218304/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.967131:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575554.967133:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575554.967135:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb000.
02000000:00000001:3.0:1788575554.967138:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575554.967140:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575554.967142:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575554.967145:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575554.967147:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5c00.
02000000:00000001:3.0:1788575554.967150:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575554.967152:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575554.967153:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:3.0:1788575554.967157:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575554.967158:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.967160:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575554.967164:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393218432/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575554.967171:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575554.967173:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575554.967175:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa000.
02000000:00000001:3.0:1788575554.967177:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575554.967178:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575554.967179:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575554.967181:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575554.967182:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:3.0:1788575554.967184:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575554.967185:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575554.967186:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:3.0:1788575554.967188:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575554.967190:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.967192:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575554.967195:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575554.967196:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575554.967198:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575554.967200:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575555.349493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575555.349502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575555.349509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575555.349513:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575555.349515:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575555.349520:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575555.349523:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575555.349525:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575555.349527:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575555.349529:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575555.349532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575555.349533:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575555.349534:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575555.349535:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575555.349536:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575555.349537:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575555.349538:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575555.349540:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575555.349591:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575555.349596:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575555.349599:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575555.477566:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575555.477577:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575555.477580:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575555.477587:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575555.477592:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575555.477598:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575555.477601:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575555.477604:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575555.477608:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575555.925521:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307159481, at 4307159488
00000100:00000040:1.0:1788575555.925521:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307159481, at 4307159488
00000100:00000010:1.0:1788575555.925540:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:0.0:1788575555.925542:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575556.373537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575556.373547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575556.373554:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575556.373555:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575556.373560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575556.373561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575556.373626:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575556.373629:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575556.373632:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575556.501481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575556.501491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575556.501494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575556.501501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575556.501505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575556.501510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575556.501513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575556.501516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575556.501520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575557.397537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575557.397548:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575557.397551:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575557.397555:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575557.397561:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575557.397564:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575557.397567:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575557.397569:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575557.397574:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575557.397575:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575557.397579:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575557.397580:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575557.397628:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575557.397633:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575557.397636:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575557.525577:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575557.525588:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575557.525590:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575557.525598:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575557.525602:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575557.525607:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575557.525610:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575557.525612:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575557.525615:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788575558.037538:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575558.037545:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788575558.037550:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575558.037722:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575558.037743:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575558.037747:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575558.037754:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575558.037758:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575558.421542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575558.421549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575558.421553:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575558.421554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575558.421557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575558.421558:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575558.421619:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575558.421622:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575558.421626:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575558.549477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575558.549485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575558.549487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575558.549492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575558.549496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575558.549500:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575558.549503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575558.549505:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575558.549507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575559.445523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575559.445529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575559.445535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575559.445536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575559.445539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575559.445540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575559.445623:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575559.445629:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575559.445634:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575559.573509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575559.573517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575559.573519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575559.573525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575559.573529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575559.573532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575559.573535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575559.573538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575559.573542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575560.085508:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575560.085522:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575560.085525:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575560.085528:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575560.085531:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575560.085537:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12491
00000400:00000200:3.0:1788575560.085548:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575560.085553:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575560.085555:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575560.085557:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085560:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575560.085564:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085566:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575560.085569:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575560.085570:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575560.085573:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575560.085575:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575560.085577:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575560.085579:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575560.085582:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085584:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575560.085585:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085587:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575560.085589:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575560.085591:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085593:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575560.085596:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575560.085600:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575560.085602:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575560.085604:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575560.085606:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575560.085614:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000020:00000040:3.0:1788575560.085616:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575560.085620:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575560.085625:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575560.085628:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575560.085630:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575560.085633:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575560.085638:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa000.
00000100:00000001:3.0:1788575560.085643:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575560.085649:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575560.085655:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575560.085658:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575560.085661:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575560.085676:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575560.085678:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575560.085680:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575560.085681:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575560.085682:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575560.085684:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575560.085686:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12491
00000400:00000200:3.0:1788575560.085688:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575560.085692:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575560.085694:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575560.085696:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085698:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575560.085699:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575560.085700:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085706:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575560.085707:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575560.085708:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1788575560.085709:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575560.085714:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575560.085715:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788575560.085715:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575560.085723:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575560.085724:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575560.085726:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085728:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575560.085734:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575560.085736:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085743:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575560.085748:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575560.085754:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575560.085765:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575560.085775:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575560.085782:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575560.085789:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1788575560.085789:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.085796:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575560.085797:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575560.085804:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575560.085805:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575560.085811:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1788575560.085812:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575560.085826:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575560.085827:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575560.085844:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000100:00100000:2.0:1788575560.085846:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393218560:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1788575560.085850:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575560.085852:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575560.085854:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575560.085856:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575560.085858:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575560.085859:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575560.085860:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575560.085863:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575560.085863:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575560.085875:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575560.085876:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
00000100:00000001:3.0:1788575560.085881:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575560.085884:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575560.085888:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575560.085888:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575560.085903:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575560.085904:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575560.085913:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575560.085914:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000040:3.0:1788575560.085919:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575560.085922:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575560.085937:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393218560, portal 25
00000100:00000001:3.0:1788575560.085937:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.085940:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575560.085943:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00080000:3.0:1788575560.085945:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575560.085948:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000040:2.0:1788575560.085951:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575585 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575560.085952:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575560.085956:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575560.085968:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575560.085978:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12491
00000100:00000001:2.0:1788575560.085982:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:3.0:1788575560.085987:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575560.085988:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575560.085991:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000400:00000200:3.0:1788575560.085992:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575560.086000:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000200:2.0:1788575560.086002:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393218560, offset 0
00000100:00000040:3.0:1788575560.086006:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.086009:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575560.086013:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575560.086027:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.086029:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575560.086031:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575560.086032:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575560.086032:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575560.086037:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575560.086040:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575560.086042:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575560.086048:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575560.086049:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575560.086058:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575560.086061:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.086064:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575560.086065:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:3.0:1788575560.086075:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575560.086081:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1788575560.086082:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575560.086087:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575560.086090:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1788575560.086092:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575560.086096:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575560.086097:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1788575560.086108:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575560.086111:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575560.086113:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575560.086115:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575560.086121:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330963:330963:256:4294967295] 192.168.202.110@tcp LPNI seq info [330920:330920:8:4294967295]
00000008:00000001:3.0:1788575560.086122:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575560.086135:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575560.086142:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1788575560.086144:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575560.086147:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575560.086149:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575560.086154:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575560.086161:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1788575560.086162:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575560.086170:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:2.0:1788575560.086175:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575560.086176:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575560.086191:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
02000000:00000010:3.0:1788575560.086192:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788575560.086206:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575560.086212:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:3.0:1788575560.086218:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1788575560.086219:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575560.086240:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575560.086241:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575560.086249:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575560.086250:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:3.0:1788575560.086261:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:2.0:1788575560.086270:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575560.086277:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000100:00000001:3.0:1788575560.086281:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575560.086285:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1788575560.086285:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575560.086297:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1788575560.086297:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575560.086308:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000100:00000040:3.0:1788575560.086311:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12501)
00000400:00000010:2.0:1788575560.086312:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575560.086319:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575560.086323:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575585 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086332:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575560.086335:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575585 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086341:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.086343:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575560.086348:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575560.086353:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575560.086355:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307164601
00000100:00000010:2.0:1788575560.086360:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cc60.
00000100:00000001:2.0:1788575560.086367:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575560.086371:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307164601
00000100:00000010:2.0:1788575560.086373:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cba0.
00000100:00000001:2.0:1788575560.086375:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575560.086378:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575560.086380:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.086382:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575560.086385:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575560/real 1788575560]  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575560.086424:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575560.086425:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575560.086427:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575560.086429:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575560.086430:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575560.086432:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575560.086435:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575560.086441:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086445:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575560.086446:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086448:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575560.086449:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575560.086451:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086456:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575560.086459:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086463:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575560.086465:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086466:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575560.086467:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086469:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575560.086470:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575560.086473:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575560.086475:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575560.086477:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575560.086480:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575560.086486:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393218560:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575560.086489:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086490:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575560.086492:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393218560/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086495:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575560.086501:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575560.086504:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575560.086506:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575560.086507:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575560.086509:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575560.086511:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575560.086513:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:2.0:1788575560.086516:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575560.086518:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575560.086519:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:2.0:1788575560.086522:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575560.086524:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086527:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086535:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575560.086538:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.086540:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575560.086542:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575560.086554:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575560.086560:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086567:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575560.086569:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575560.086571:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575560.086573:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575560.086575:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575560.086577:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575560.086579:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575560.086581:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575560.086582:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575560.086583:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575560.086585:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575560.086587:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575560.086590:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575560.086592:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575560.086594:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575560.086596:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393218688, portal 10
00000100:00000001:2.0:1788575560.086597:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575560.086599:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:2.0:1788575560.086605:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575585 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086611:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575560.086613:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575560.086615:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575560.086617:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393218688, offset 0
00000400:00000200:2.0:1788575560.086621:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575560.086625:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575560.086628:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575560.086631:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575560.086633:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575560.086635:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575560.086638:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575560.086640:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330964:330964:256:4294967295] 192.168.202.110@tcp LPNI seq info [330921:330921:8:4294967295]
00000400:00000200:2.0:1788575560.086645:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575560.086651:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575560.086655:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575560.086659:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575560.086662:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575560.086664:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575560.086667:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575560.086668:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575560.086671:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575560.086672:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575560.086674:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575560.086676:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575560.086678:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575560.086680:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575560.086681:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575560.086684:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575585 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086689:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575560.086691:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575585 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086695:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.086696:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575560.086698:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575560.086701:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575560.086704:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575560.086706:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307163687
00000100:00000001:2.0:1788575560.086710:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307164601
00000100:00000001:2.0:1788575560.086711:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307163687
00000100:00000001:2.0:1788575560.086714:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307164601
00000100:00000001:2.0:1788575560.086716:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575560.086718:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.086720:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575560.086723:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575560/real 1788575560]  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575560.086739:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575560.086740:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575560.086742:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575560.086743:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575560.086745:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575560.086746:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575560.086748:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575560.086753:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086758:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575560.086758:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086759:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575560.086760:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575560.086762:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086766:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575560.086768:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086772:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575560.086774:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086775:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575560.086775:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086776:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575560.086778:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575560.086779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575560.086781:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575560.086782:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575560.086785:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575560.086790:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575560.086792:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575560.086794:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086798:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575560.086799:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575560.086800:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575560.086801:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575560.086802:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575560.086804:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575560.086806:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575560.086807:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575560.086809:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575560.086810:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575560.086812:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575560.086813:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575560.086817:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575560.086819:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575560.086822:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575560.086825:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393218816, portal 4
00000100:00000001:2.0:1788575560.086827:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575560.086828:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788575560.086834:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575585 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086840:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575560.086843:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575560.086845:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575560.086848:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393218816, offset 0
00000400:00000200:2.0:1788575560.086852:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575560.086855:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575560.086859:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575560.086862:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575560.086864:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575560.086867:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575560.086869:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575560.086871:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330965:330965:256:4294967295] 192.168.202.110@tcp LPNI seq info [330922:330922:8:4294967295]
00000400:00000200:2.0:1788575560.086876:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575560.086881:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575560.086886:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575560.086889:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575560.086893:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575560.086895:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575560.086898:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575560.086900:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575560.086902:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575560.086903:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575560.086905:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575560.086907:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575560.086910:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575560.086911:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575560.086912:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575560.086915:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575585 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086920:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575560.086922:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575585 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086931:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.086932:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575560.086934:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575560.086936:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575560.086938:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575560.086940:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307163833
00000100:00000001:2.0:1788575560.086942:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307164601
00000100:00000001:2.0:1788575560.086943:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307163833
00000100:00000001:2.0:1788575560.086945:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307164601
00000100:00000001:2.0:1788575560.086946:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575560.086947:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.086948:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575560.086950:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575560/real 1788575560]  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575560.086955:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575560.086956:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575560.086958:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575560.086960:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575560.086961:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575560.086964:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575560.086967:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575560.086975:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086981:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575560.086983:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.086985:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575560.086986:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575560.086990:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.086997:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575560.087001:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.087006:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575560.087008:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.087009:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575560.087010:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.087011:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575560.087012:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575560.087014:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575560.087027:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575560.087028:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575560.087031:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575560.087036:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393218816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575560.087038:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.087040:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575560.087042:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393218688/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.087046:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575560.087047:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575560.087048:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575560.087050:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575560.087051:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575560.087052:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575560.087053:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575560.087054:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:2.0:1788575560.087056:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575560.087057:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575560.087058:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575560.087060:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575560.087061:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.087062:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575560.087064:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393218816/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575585 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575560.087072:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575560.087072:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575560.087073:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575560.087074:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575560.087075:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575560.087076:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575560.087077:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575560.087077:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575560.087078:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575560.087079:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575560.087080:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575560.087081:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575560.087081:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.087082:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575560.087084:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575560.087085:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575560.087086:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575560.087087:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575560.469511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575560.469518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575560.469523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575560.469525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575560.469528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575560.469530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575560.469593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575560.469597:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575560.469600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575560.597466:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575560.597473:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575560.597475:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575560.597479:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575560.597481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575560.597484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575560.597486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575560.597488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575560.597490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575561.045500:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307164601, at 4307164608
00000100:00000040:1.0:1788575561.045500:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307164601, at 4307164608
00000100:00000010:1.0:1788575561.045514:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000100:00000010:0.0:1788575561.045526:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00010000:00000001:3.0:1788575561.493530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575561.493537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575561.493541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575561.493542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575561.493544:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575561.493545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575561.493608:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575561.493611:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575561.493614:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575561.621511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575561.621519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575561.621521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575561.621527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575561.621531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575561.621535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575561.621538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575561.621540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575561.621543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575562.517543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575562.517552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575562.517560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575562.517561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575562.517565:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575562.517566:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575562.517633:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575562.517638:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575562.517642:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575562.645572:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575562.645582:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575562.645585:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575562.645601:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575562.645609:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575562.645614:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575562.645617:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575562.645620:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575562.645624:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575563.541478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575563.541485:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575563.541490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575563.541492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575563.541495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575563.541496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575563.541568:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575563.541572:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575563.541576:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575563.669521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575563.669529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575563.669530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575563.669536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575563.669539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575563.669543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575563.669546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575563.669548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575563.669550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575564.565499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575564.565506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575564.565510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575564.565512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575564.565514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575564.565515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575564.565559:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575564.565564:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575564.565568:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575564.693541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575564.693550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575564.693551:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575564.693557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575564.693560:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575564.693564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575564.693565:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575564.693568:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575564.693570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575565.205510:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575565.205524:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575565.205527:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575565.205532:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575565.205538:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575565.205547:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12496
00000400:00000200:0.0:1788575565.205559:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575565.205566:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575565.205569:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575565.205573:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205576:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575565.205579:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205581:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575565.205586:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575565.205588:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575565.205591:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575565.205595:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575565.205598:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575565.205600:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575565.205603:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205605:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575565.205607:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205609:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575565.205612:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575565.205616:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205619:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575565.205622:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575565.205626:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575565.205629:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575565.205632:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575565.205635:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575565.205646:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575565.205649:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575565.205652:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575565.205654:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575565.205656:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575565.205659:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575565.205663:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575565.205670:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575565.205677:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575565.205685:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575565.205693:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575565.205697:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575565.205701:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575565.205718:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575565.205721:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575565.205724:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575565.205725:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575565.205726:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575565.205728:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575565.205730:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12496
00000400:00000200:0.0:1788575565.205734:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575565.205737:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575565.205739:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575565.205741:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205743:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575565.205743:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1788575565.205746:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205749:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575565.205749:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788575565.205753:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575565.205754:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1788575565.205756:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575565.205758:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575565.205760:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575565.205762:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575565.205763:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1788575565.205764:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575565.205769:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205772:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575565.205775:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205779:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575565.205782:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575565.205786:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205790:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575565.205794:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575565.205797:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788575565.205799:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575565.205800:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575565.205805:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575565.205807:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575565.205808:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575565.205811:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:0.0:1788575565.205813:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575565.205814:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788575565.205818:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575565.205822:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575565.205822:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393218944:192.168.202.110@tcp:250:kworker.0
00000100:00000010:0.0:1788575565.205827:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000100:00000001:2.0:1788575565.205828:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1788575565.205829:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575565.205832:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575565.205832:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575565.205833:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575565.205836:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575565.205836:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575565.205839:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575565.205840:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1788575565.205842:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575565.205846:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
02000000:00000010:2.0:1788575565.205847:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
00000100:00000001:0.0:1788575565.205849:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575565.205851:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575565.205855:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575565.205860:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575565.205866:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1788575565.205867:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000001:0.0:1788575565.205869:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575565.205873:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575565.205878:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575565.205879:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575565.205881:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575565.205882:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575565.205883:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575565.205885:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:2.0:1788575565.205886:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00080000:0.0:1788575565.205887:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12496
00000400:00000200:0.0:1788575565.205892:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575565.205894:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575565.205895:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000200:2.0:1788575565.205896:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393218944, portal 25
00000100:00000040:0.0:1788575565.205897:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205900:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575565.205900:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575565.205903:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000040:0.0:1788575565.205904:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205908:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575565.205909:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575565.205911:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575565.205913:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575565.205913:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575590 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575565.205914:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575565.205918:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575565.205922:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575565.205928:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575565.205929:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788575565.205930:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575565.205934:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575565.205935:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575565.205936:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575565.205940:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:2.0:1788575565.205940:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00080000:0.0:1788575565.205945:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:2.0:1788575565.205946:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393218944, offset 0
00000100:00000001:0.0:1788575565.205947:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575565.205950:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575565.205952:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575565.205955:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575565.205956:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575565.205960:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575565.205961:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575565.205963:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575565.205964:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575565.205966:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000400:00000200:2.0:1788575565.205967:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1788575565.205969:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575565.205971:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575565.205973:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575565.205975:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575565.205976:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575565.205977:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575565.205979:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000400:00000200:2.0:1788575565.205980:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788575565.205982:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575565.205987:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:0.0:1788575565.205988:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575565.205994:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1788575565.205995:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575565.205997:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575565.206000:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1788575565.206001:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575565.206006:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788575565.206011:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575565.206014:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330966:330966:256:4294967295] 192.168.202.110@tcp LPNI seq info [330923:330923:8:4294967295]
00000100:00000040:0.0:1788575565.206015:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575565.206022:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575565.206025:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575565.206028:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12506)
00000400:00000200:2.0:1788575565.206039:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575565.206045:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575565.206050:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575565.206055:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575565.206058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575565.206062:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575565.206065:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575565.206068:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575565.206070:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575565.206076:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575565.206079:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575565.206083:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575565.206085:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575565.206089:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575565.206092:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575590 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206109:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575565.206112:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575590 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206117:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575565.206118:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575565.206124:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575565.206127:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575565.206129:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307169721
00000100:00000010:2.0:1788575565.206134:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c1e0.
00000100:00000001:2.0:1788575565.206140:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575565.206146:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307169721
00000100:00000010:2.0:1788575565.206148:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c7e0.
00000100:00000001:2.0:1788575565.206150:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575565.206152:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575565.206154:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575565.206156:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575565.206159:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575565/real 1788575565]  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575565.206171:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575565.206172:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575565.206174:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575565.206175:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575565.206177:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575565.206178:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575565.206181:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575565.206186:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206191:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575565.206192:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206193:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575565.206194:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575565.206197:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206202:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575565.206205:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206210:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575565.206212:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206213:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575565.206213:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206215:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575565.206216:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575565.206218:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575565.206221:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575565.206223:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575565.206226:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575565.206233:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393218944:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575565.206238:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206240:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575565.206243:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393218944/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206249:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575565.206252:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575565.206255:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575565.206258:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575565.206260:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575565.206261:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575565.206264:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575565.206267:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:2.0:1788575565.206271:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575565.206274:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575565.206275:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:2.0:1788575565.206279:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575565.206280:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206282:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206289:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575565.206292:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575565.206295:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575565.206296:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575565.206297:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575565.206300:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206304:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575565.206305:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575565.206307:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575565.206308:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575565.206310:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575565.206312:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219072:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575565.206314:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575565.206315:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575565.206316:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575565.206318:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575565.206320:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575565.206321:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575565.206323:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575565.206325:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575565.206328:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575565.206330:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393219072, portal 10
00000100:00000001:2.0:1788575565.206331:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575565.206332:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
00000100:00000040:2.0:1788575565.206337:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575590 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206341:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575565.206342:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575565.206343:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575565.206345:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393219072, offset 0
00000400:00000200:2.0:1788575565.206347:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575565.206349:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575565.206352:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575565.206354:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575565.206355:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575565.206357:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575565.206358:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575565.206360:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330967:330967:256:4294967295] 192.168.202.110@tcp LPNI seq info [330924:330924:8:4294967295]
00000400:00000200:2.0:1788575565.206363:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575565.206367:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575565.206371:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575565.206373:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575565.206376:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575565.206378:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575565.206380:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575565.206382:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575565.206385:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575565.206386:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575565.206388:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575565.206413:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575565.206415:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575565.206416:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575565.206418:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575565.206421:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575590 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206426:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575565.206427:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575590 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206432:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575565.206433:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575565.206435:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575565.206437:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575565.206439:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575565.206440:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307168807
00000100:00000001:2.0:1788575565.206443:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307169721
00000100:00000001:2.0:1788575565.206444:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307168807
00000100:00000001:2.0:1788575565.206445:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307169721
00000100:00000001:2.0:1788575565.206447:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575565.206448:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575565.206450:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575565.206452:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575565/real 1788575565]  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575565.206460:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575565.206461:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575565.206462:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575565.206464:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575565.206464:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575565.206466:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575565.206467:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575565.206472:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206476:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575565.206477:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206478:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575565.206479:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575565.206481:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206484:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575565.206486:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206490:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575565.206491:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206492:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575565.206493:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206494:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575565.206495:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575565.206496:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575565.206498:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575565.206500:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575565.206502:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575565.206507:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219072:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575565.206509:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575565.206511:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206515:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575565.206515:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575565.206517:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575565.206518:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575565.206519:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575565.206520:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219200:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575565.206522:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575565.206523:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575565.206524:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575565.206525:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575565.206528:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575565.206529:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575565.206531:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575565.206533:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575565.206535:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575565.206537:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393219200, portal 4
00000100:00000001:2.0:1788575565.206538:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575565.206539:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:2.0:1788575565.206543:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575590 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206547:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575565.206548:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575565.206550:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575565.206551:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393219200, offset 0
00000400:00000200:2.0:1788575565.206554:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575565.206556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575565.206558:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575565.206560:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575565.206561:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575565.206563:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575565.206564:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575565.206566:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330968:330968:256:4294967295] 192.168.202.110@tcp LPNI seq info [330925:330925:8:4294967295]
00000400:00000200:2.0:1788575565.206569:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575565.206573:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575565.206575:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575565.206577:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575565.206579:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575565.206580:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575565.206583:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575565.206584:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575565.206587:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575565.206588:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575565.206590:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575565.206592:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575565.206594:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575565.206595:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575565.206596:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575565.206598:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575590 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206602:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575565.206604:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575590 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206608:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575565.206609:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575565.206610:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575565.206612:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575565.206613:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575565.206615:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307168953
00000100:00000001:2.0:1788575565.206616:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307169721
00000100:00000001:2.0:1788575565.206618:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307168953
00000100:00000001:2.0:1788575565.206619:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307169721
00000100:00000001:2.0:1788575565.206620:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575565.206621:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575565.206622:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575565.206624:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575565/real 1788575565]  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575565.206628:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575565.206628:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575565.206629:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575565.206630:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575565.206632:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575565.206633:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575565.206635:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575565.206639:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206643:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575565.206643:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206644:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575565.206645:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575565.206647:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206650:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575565.206652:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206655:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575565.206657:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206658:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575565.206659:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206660:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575565.206661:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575565.206663:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575565.206665:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575565.206666:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575565.206668:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575565.206672:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219200:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575565.206674:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206675:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575565.206676:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393219072/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206680:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575565.206681:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575565.206682:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575565.206683:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575565.206684:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575565.206685:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575565.206686:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575565.206687:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:2.0:1788575565.206689:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575565.206690:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575565.206690:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:2.0:1788575565.206692:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575565.206693:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206694:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575565.206696:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393219200/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575590 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575565.206704:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575565.206705:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575565.206706:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575565.206707:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575565.206707:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575565.206708:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575565.206709:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575565.206710:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788575565.206711:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575565.206711:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575565.206712:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:2.0:1788575565.206713:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575565.206714:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206715:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575565.206717:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575565.206717:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575565.206718:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575565.206720:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575565.589549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575565.589556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575565.589560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575565.589564:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575565.589566:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575565.589568:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575565.589571:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575565.589572:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575565.589574:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575565.589575:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575565.589579:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575565.589580:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575565.589581:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575565.589582:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575565.589584:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575565.589584:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575565.589586:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575565.589587:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575565.589650:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575565.589654:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575565.589656:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575565.717455:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575565.717463:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575565.717465:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575565.717470:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575565.717475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575565.717479:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575565.717482:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575565.717486:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575565.717489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575566.165470:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307169721, at 4307169728
00000100:00000010:1.0:1788575566.165487:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000100:00000040:0.0:1788575566.165509:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307169721, at 4307169728
00000100:00000010:0.0:1788575566.165520:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00010000:00000001:3.0:1788575566.613510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575566.613515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575566.613520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575566.613521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575566.613524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575566.613525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575566.613581:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575566.613583:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575566.613585:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575566.741463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575566.741471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575566.741472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575566.741477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575566.741480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575566.741484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575566.741486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575566.741488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575566.741491:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575567.637544:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575567.637553:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575567.637555:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575567.637558:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575567.637563:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575567.637564:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575567.637566:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575567.637568:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575567.637570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575567.637571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575567.637575:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575567.637576:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575567.637639:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575567.637644:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575567.637648:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575567.765536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575567.765547:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575567.765549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575567.765555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575567.765558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575567.765563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575567.765565:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575567.765567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575567.765569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575568.661501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575568.661507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575568.661512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575568.661513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575568.661515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575568.661516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575568.661569:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575568.661572:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575568.661574:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575568.789508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575568.789515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575568.789517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575568.789522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575568.789526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575568.789530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575568.789532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575568.789535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575568.789538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575569.685504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575569.685509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575569.685513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575569.685514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575569.685516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575569.685517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575569.685576:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575569.685580:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575569.685583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575569.813495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575569.813501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575569.813502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575569.813506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575569.813509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575569.813512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575569.813514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575569.813515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575569.813517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575570.325496:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575570.325503:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575570.325505:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575570.325508:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575570.325511:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575570.325516:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12501
00000400:00000200:3.0:1788575570.325523:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575570.325527:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575570.325529:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575570.325531:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325533:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575570.325535:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325537:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575570.325540:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575570.325541:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575570.325543:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575570.325545:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575570.325547:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575570.325548:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575570.325549:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325550:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575570.325552:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325553:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575570.325555:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575570.325557:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325559:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575570.325561:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575570.325563:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575570.325564:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575570.325566:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575570.325568:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575570.325575:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000020:00000040:3.0:1788575570.325577:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575570.325580:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575570.325581:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575570.325583:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575570.325585:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575570.325588:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575570.325593:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575570.325598:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575570.325604:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575570.325610:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575570.325613:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575570.325616:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575570.325630:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575570.325632:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575570.325634:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575570.325635:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575570.325636:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575570.325638:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575570.325640:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12501
00000400:00000200:3.0:1788575570.325641:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575570.325643:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575570.325644:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575570.325646:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325648:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575570.325649:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325651:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575570.325652:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575570.325654:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575570.325655:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575570.325656:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575570.325658:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575570.325659:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575570.325660:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575570.325661:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325665:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575570.325666:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575570.325666:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325669:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575570.325671:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788575570.325672:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575570.325676:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325677:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575570.325679:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575570.325682:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575570.325684:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575570.325687:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575570.325690:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575570.325694:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575570.325698:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.325702:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575570.325702:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.325707:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575570.325707:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788575570.325710:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575570.325712:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575570.325713:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575570.325714:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575570.325714:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575570.325716:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575570.325719:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575570.325720:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:3.0:1788575570.325722:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575570.325725:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575570.325726:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575570.325733:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:2.0:1788575570.325734:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393219328:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575570.325743:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575570.325745:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000001:2.0:1788575570.325748:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1788575570.325748:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575570.325751:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575570.325753:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575570.325759:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575570.325760:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:2.0:1788575570.325764:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:3.0:1788575570.325764:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575570.325767:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575570.325768:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575570.325769:0:188751:0:(import.c:520:import_select_connection()) Process entered
02000000:00000010:2.0:1788575570.325771:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
00000100:00080000:3.0:1788575570.325771:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12501
00000400:00000200:3.0:1788575570.325776:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575570.325777:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575570.325785:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575570.325786:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:2.0:1788575570.325790:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00080000:3.0:1788575570.325790:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575570.325796:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325799:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575570.325800:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000040:3.0:1788575570.325802:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325806:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:2.0:1788575570.325809:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393219328, portal 25
00000020:00000001:3.0:1788575570.325809:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575570.325812:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:2.0:1788575570.325815:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000020:00000001:3.0:1788575570.325816:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575570.325819:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000020:00000040:3.0:1788575570.325820:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575570.325825:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575570.325829:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575570.325831:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575570.325834:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325838:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575570.325840:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325844:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575570.325848:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788575570.325852:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1788575570.325852:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325856:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575570.325858:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575570.325860:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575570.325861:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575570.325864:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575570.325865:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000001:3.0:1788575570.325866:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575570.325868:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575570.325871:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575570.325872:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393219328, offset 0
00000100:00000001:3.0:1788575570.325872:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575570.325877:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000020:00000040:3.0:1788575570.325880:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575570.325882:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575570.325884:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575570.325886:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575570.325886:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575570.325888:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575570.325891:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575570.325894:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575570.325897:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575570.325898:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1788575570.325902:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575570.325908:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575570.325910:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788575570.325910:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575570.325916:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575570.325920:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788575570.325923:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575570.325926:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1788575570.325926:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575570.325930:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575570.325934:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1788575570.325936:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12511)
00000400:00000200:2.0:1788575570.325939:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575570.325944:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330969:330969:256:4294967295] 192.168.202.110@tcp LPNI seq info [330926:330926:8:4294967295]
00000400:00000200:2.0:1788575570.325949:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575570.325956:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575570.325960:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575570.325965:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575570.325969:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575570.325972:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575570.325976:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575570.325978:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575570.325981:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575570.325983:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575570.325988:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575570.325991:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575570.325995:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575570.325996:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575570.326000:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575570.326003:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326009:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575570.326011:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575595 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326016:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.326017:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575570.326021:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575570.326023:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575570.326025:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307174841
00000100:00000010:2.0:1788575570.326029:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c300.
00000100:00000001:2.0:1788575570.326034:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575570.326043:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307174841
00000100:00000010:2.0:1788575570.326045:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c2a0.
00000100:00000001:2.0:1788575570.326047:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575570.326049:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575570.326051:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.326053:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575570.326056:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575570/real 1788575570]  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575570.326067:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575570.326068:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575570.326070:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575570.326072:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575570.326073:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575570.326074:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575570.326076:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575570.326081:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326085:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575570.326086:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326087:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575570.326089:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575570.326091:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326095:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575570.326097:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326102:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575570.326103:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326104:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575570.326105:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326106:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575570.326108:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575570.326109:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575570.326112:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575570.326113:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575570.326115:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575570.326120:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393219328:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575570.326123:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326124:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575570.326126:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393219328/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326129:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575570.326131:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575570.326133:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575570.326135:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575570.326136:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575570.326137:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575570.326138:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575570.326140:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575570.326142:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575570.326144:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575570.326145:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575570.326147:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575570.326147:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326149:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326154:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575570.326155:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.326157:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575570.326158:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575570.326158:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575570.326160:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326165:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575570.326165:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575570.326167:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575570.326168:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575570.326169:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575570.326171:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219456:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575570.326173:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575570.326174:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575570.326198:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575570.326200:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575570.326202:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575570.326204:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575570.326206:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575570.326208:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575570.326210:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575570.326212:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393219456, portal 10
00000100:00000001:2.0:1788575570.326214:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575570.326214:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788575570.326220:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326224:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575570.326225:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575570.326226:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575570.326228:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393219456, offset 0
00000400:00000200:2.0:1788575570.326230:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575570.326232:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575570.326234:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575570.326236:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575570.326238:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575570.326239:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575570.326241:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575570.326242:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330970:330970:256:4294967295] 192.168.202.110@tcp LPNI seq info [330927:330927:8:4294967295]
00000400:00000200:2.0:1788575570.326245:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575570.326249:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575570.326251:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575570.326253:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575570.326255:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575570.326257:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575570.326260:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575570.326261:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575570.326263:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575570.326265:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575570.326266:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575570.326268:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575570.326270:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575570.326271:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575570.326273:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575570.326275:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326279:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575570.326281:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575595 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326289:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.326291:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575570.326292:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575570.326295:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575570.326297:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575570.326298:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307173926
00000100:00000001:2.0:1788575570.326300:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307174841
00000100:00000001:2.0:1788575570.326302:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307173926
00000100:00000001:2.0:1788575570.326303:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307174841
00000100:00000001:2.0:1788575570.326304:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575570.326305:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.326306:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575570.326308:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575570/real 1788575570]  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575570.326313:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575570.326314:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575570.326315:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575570.326316:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575570.326317:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575570.326319:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575570.326320:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575570.326325:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326328:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575570.326329:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326330:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575570.326331:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575570.326332:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326336:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575570.326338:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326342:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575570.326343:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326344:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575570.326345:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326346:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575570.326347:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575570.326349:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575570.326351:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575570.326352:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575570.326354:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575570.326358:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219456:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575570.326360:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575570.326362:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326366:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575570.326367:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575570.326368:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575570.326369:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575570.326370:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575570.326372:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575570.326373:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575570.326374:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575570.326375:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575570.326376:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575570.326378:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575570.326380:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575570.326382:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575570.326383:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575570.326385:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575570.326386:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393219584, portal 4
00000100:00000001:2.0:1788575570.326388:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575570.326388:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575570.326415:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326419:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575570.326421:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575570.326423:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575570.326424:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393219584, offset 0
00000400:00000200:2.0:1788575570.326426:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575570.326429:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575570.326431:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575570.326433:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575570.326435:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575570.326436:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575570.326438:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575570.326440:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330971:330971:256:4294967295] 192.168.202.110@tcp LPNI seq info [330928:330928:8:4294967295]
00000400:00000200:2.0:1788575570.326449:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575570.326455:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575570.326459:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575570.326462:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575570.326466:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575570.326468:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575570.326472:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575570.326475:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575570.326479:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575570.326480:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:2.0:1788575570.326483:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575570.326486:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575570.326489:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:2.0:1788575570.326491:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:2.0:1788575570.326493:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575570.326496:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326503:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575570.326506:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575595 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326513:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.326514:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575570.326517:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575570.326519:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575570.326522:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575570.326524:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307174073
00000100:00000001:2.0:1788575570.326526:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307174841
00000100:00000001:2.0:1788575570.326528:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307174073
00000100:00000001:2.0:1788575570.326529:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307174841
00000100:00000001:2.0:1788575570.326531:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575570.326532:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.326533:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575570.326536:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575570/real 1788575570]  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575570.326540:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575570.326541:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575570.326542:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575570.326544:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575570.326545:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575570.326546:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575570.326548:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575570.326552:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326556:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575570.326556:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326557:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575570.326558:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575570.326560:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326563:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575570.326566:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326569:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575570.326571:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326572:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575570.326573:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326574:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575570.326576:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575570.326579:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575570.326582:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575570.326584:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575570.326588:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575570.326594:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575570.326598:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326599:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575570.326602:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393219456/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326608:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575570.326610:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575570.326612:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575570.326614:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575570.326616:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575570.326617:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575570.326619:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575570.326621:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575570.326623:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575570.326624:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575570.326626:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575570.326628:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575570.326629:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326631:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575570.326634:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393219584/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575595 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575570.326640:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575570.326641:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575570.326643:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575570.326645:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575570.326647:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575570.326648:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575570.326650:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575570.326651:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575570.326653:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575570.326654:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575570.326655:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575570.326656:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575570.326657:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326658:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575570.326661:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575570.326661:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575570.326663:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575570.326664:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575570.709495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575570.709500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575570.709504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575570.709505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575570.709507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575570.709507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575570.709605:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575570.709609:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575570.709612:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575570.837500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575570.837508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575570.837510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575570.837516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575570.837520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575570.837524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575570.837526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575570.837529:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575570.837532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575571.285495:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307174841, at 4307174848
00000100:00000040:1.0:1788575571.285495:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307174841, at 4307174848
00000100:00000010:0.0:1788575571.285511:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00000100:00000010:1.0:1788575571.285513:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00010000:00000001:3.0:1788575571.733507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575571.733512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575571.733516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575571.733517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575571.733519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575571.733521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575571.733563:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575571.733567:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575571.733570:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575571.861487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575571.861493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575571.861494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575571.861498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575571.861500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575571.861504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575571.861505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575571.861507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575571.861509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575572.757496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575572.757501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575572.757504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575572.757505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575572.757507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575572.757508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575572.757558:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575572.757561:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575572.757563:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575572.885681:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575572.885685:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575572.885687:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575572.885690:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575572.885693:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575572.885695:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575572.885697:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575572.885699:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575572.885701:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575573.781527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575573.781533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575573.781538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575573.781540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575573.781543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575573.781544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575573.781614:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575573.781618:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575573.781621:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575573.909523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575573.909530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575573.909532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575573.909538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575573.909541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575573.909546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575573.909548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575573.909551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575573.909553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575574.805487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575574.805493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575574.805498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575574.805501:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575574.805502:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575574.805504:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575574.805506:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575574.805508:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575574.805509:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575574.805510:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575574.805513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575574.805513:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575574.805514:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575574.805515:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575574.805516:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575574.805517:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575574.805518:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575574.805519:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575574.805564:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575574.805568:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575574.805572:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575574.933509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575574.933515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575574.933516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575574.933520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575574.933522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575574.933525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575574.933527:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575574.933529:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575574.933530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575575.445602:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575575.445613:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575575.445615:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575575.445619:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575575.445624:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575575.445630:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12506
00000400:00000200:0.0:1788575575.445638:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575575.445643:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575575.445646:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575575.445648:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575575.445651:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575575.445652:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575575.445654:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575575.445657:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575575.445658:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575575.445660:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575575.445662:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575575.445663:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575575.445665:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575575.445667:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575575.445668:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575575.445669:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575575.445671:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575575.445673:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575575.445675:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575575.445676:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575575.445678:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575575.445680:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575575.445682:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575575.445683:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575575.445685:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575575.445692:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575575.445695:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575575.445697:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575575.445698:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575575.445701:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575575.445703:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575575.445706:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575575.445711:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788575575.445716:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575575.445722:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575575.445731:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575575.445735:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575575.445739:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575575.445760:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575575.445764:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575575.445767:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575575.445768:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575575.445770:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575575.445772:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575575.445774:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12506
00000400:00000200:0.0:1788575575.445777:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575575.445780:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575575.445782:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575575.445784:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575575.445786:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575575.445787:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575575.445788:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575575.445793:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575575.445794:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788575575.445795:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575575.445798:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575575.445800:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575575.445801:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575575.445801:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575575.445805:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575575.445808:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575575.445811:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575575.445817:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575575.445817:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575575.445819:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575575.445820:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575575.445821:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575575.445824:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575575.445824:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1788575575.445827:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575575.445829:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575575.445831:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575575.445832:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575575.445835:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575575.445836:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1788575575.445840:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393219712:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788575575.445841:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575575.445846:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.445847:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1788575575.445848:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575575.445851:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:0.0:1788575575.445852:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575575.445854:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788575575.445855:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575575.445857:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788575575.445858:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575575.445862:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575575.445864:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575575.445870:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000400:00000200:2.0:1788575575.445871:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1788575575.445875:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:2.0:1788575575.445877:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
02000000:00000001:0.0:1788575575.445879:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575575.445882:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575575.445884:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575575.445886:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1788575575.445886:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
02000000:00000001:0.0:1788575575.445889:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575575.445895:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393219712, portal 25
02000000:00000010:0.0:1788575575.445896:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:2.0:1788575575.445900:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575575.445902:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.445904:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00100000:0.0:1788575575.445910:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788575575.445914:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575575.445918:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575575.445921:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.445927:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575575.445931:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575575.445934:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65b28.
00000100:00000200:2.0:1788575575.445937:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393219712, offset 0
00000400:00000200:2.0:1788575575.445944:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575575.445950:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575575.445955:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575575.445959:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575575.445962:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575575.445965:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575575.445966:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575575.445969:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330972:330972:256:4294967295] 192.168.202.110@tcp LPNI seq info [330929:330929:8:4294967295]
00000400:00000200:2.0:1788575575.445972:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575575.445978:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575575.445983:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575575.445986:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788575575.445991:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575575.445994:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788575575.445999:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575575.446006:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575575.446013:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575575.446017:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307179961
00000100:00000010:2.0:1788575575.446021:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ca80.
00000100:00000001:0.0:1788575575.446023:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575575.446027:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575575.446028:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:0.0:1788575575.446033:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575575.446035:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1788575575.446036:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307179961
00000100:00080000:0.0:1788575575.446039:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:2.0:1788575575.446041:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c6c0.
00000100:00000001:0.0:1788575575.446045:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:2.0:1788575575.446045:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575575.446049:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1788575575.446051:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12506
00000100:00000001:2.0:1788575575.446053:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.446057:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575575.446058:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575575.446059:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1788575575.446063:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1788575575.446066:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1788575575.446067:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575575.446069:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575575.446072:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1788575575.446073:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575575.446073:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575575.446075:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788575575.446078:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575575.446081:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575575.446083:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575575.446087:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575575.446092:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575575.446093:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575575.446094:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575575.446096:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575575.446097:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575575.446100:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219840:192.168.202.110@tcp:38:kworker.0
00000020:00000001:0.0:1788575575.446101:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575575.446103:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:2.0:1788575575.446105:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1788575575.446107:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1788575575.446108:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575575.446110:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575575.446111:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
02000000:00000001:2.0:1788575575.446112:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:0.0:1788575575.446115:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
02000000:00000010:2.0:1788575575.446116:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
00000020:00000001:0.0:1788575575.446121:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
02000000:00000001:2.0:1788575575.446122:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575575.446127:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575575.446129:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:2.0:1788575575.446131:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00000001:0.0:1788575575.446134:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575575.446136:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000040:0.0:1788575575.446139:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575575.446140:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393219840, portal 10
00000100:00000001:0.0:1788575575.446144:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575575.446144:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575575.446146:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000020:00000040:0.0:1788575575.446149:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000040:2.0:1788575575.446154:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788575575.446157:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575575.446163:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.446165:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1788575575.446168:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575575.446170:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575575.446172:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65d48.
02000000:00000001:0.0:1788575575.446173:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:2.0:1788575575.446174:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393219840, offset 0
02000000:00000001:0.0:1788575575.446179:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575575.446181:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575575.446183:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788575575.446186:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:0.0:1788575575.446187:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575575.446189:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575575.446190:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575575.446192:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575575.446194:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575575.446195:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788575575.446198:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575575.446198:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575575.446202:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330973:330973:255:4294967295] 192.168.202.110@tcp LPNI seq info [330930:330930:7:4294967295]
00000100:00000010:0.0:1788575575.446204:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000400:00000200:2.0:1788575575.446207:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1788575575.446209:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575575.446213:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575575.446215:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575575.446216:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788575575.446217:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575575.446220:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575575.446223:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575575.446224:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000010:0.0:1788575575.446228:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000800:00000010:2.0:1788575575.446229:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133c00.
00000100:00000001:0.0:1788575575.446234:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575575.446235:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:0.0:1788575575.446244:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1788575575.446244:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575575.446248:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575575.446254:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1788575575.446254:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000100:00000001:0.0:1788575575.446267:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575575.446271:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575575.446277:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575575.446282:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307179046
00000100:00000001:0.0:1788575575.446283:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.446288:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307179961
00000100:00000040:0.0:1788575575.446289:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575575.446295:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307179046
00000100:00000040:0.0:1788575575.446296:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575575.446299:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307179961
00000100:00000001:2.0:1788575575.446303:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575575.446305:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.446308:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575575.446310:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12516)
00000100:00000001:2.0:1788575575.446310:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575575.446317:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575575.446318:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575575.446319:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575575.446320:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575575.446321:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575575.446325:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575575.446338:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575575.446339:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575575.446342:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575575.446343:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575575.446345:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575575.446348:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219968:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575575.446351:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575575.446353:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575575.446355:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575575.446356:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575575.446360:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575575.446361:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575575.446365:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575575.446367:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575575.446370:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575575.446373:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393219968, portal 4
00000100:00000001:2.0:1788575575.446374:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575575.446376:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:2.0:1788575575.446381:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575575.446387:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575575.446430:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575575.446433:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65000.
00000100:00000200:2.0:1788575575.446436:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393219968, offset 0
00000400:00000200:2.0:1788575575.446440:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575575.446445:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575575.446448:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575575.446450:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575575.446452:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575575.446454:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575575.446456:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575575.446459:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330974:330974:254:4294967295] 192.168.202.110@tcp LPNI seq info [330931:330931:6:4294967295]
00000400:00000200:2.0:1788575575.446463:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575575.446468:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575575.446473:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575575.446476:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788575575.446480:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575575.446483:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575575.446486:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575575.446489:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575575.446492:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575575.446494:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307179193
00000100:00000001:2.0:1788575575.446496:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307179961
00000100:00000001:2.0:1788575575.446498:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307179193
00000100:00000001:2.0:1788575575.446500:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307179961
00000100:00000001:2.0:1788575575.446502:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575575.446504:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.446506:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.446508:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575575.446511:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575575.446512:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575575.446519:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575575.446520:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575575.446522:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575575.446524:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575575.829492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575575.829499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575575.829503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575575.829504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575575.829506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575575.829507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575575.829559:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575575.829563:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575575.829566:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575575.957504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575575.957511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575575.957513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575575.957518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575575.957522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575575.957526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575575.957529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575575.957532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575575.957534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575576.405461:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307179961, at 4307179968
00000100:00000040:1.0:1788575576.405469:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307179961, at 4307179968
00000100:00000010:0.0:1788575576.405473:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000100:00000010:1.0:1788575576.405477:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00010000:00000001:3.0:1788575576.853462:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575576.853468:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575576.853470:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575576.853472:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575576.853475:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575576.853477:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575576.853478:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575576.853480:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575576.853482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575576.853483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575576.853485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575576.853486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575576.853533:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575576.853537:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575576.853540:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575576.981512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575576.981519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575576.981521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575576.981526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575576.981528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575576.981531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575576.981533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575576.981535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575576.981536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575577.877516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575577.877522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575577.877527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575577.877529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575577.877532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575577.877533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575577.877580:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575577.877583:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575577.877586:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575578.005463:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575578.005468:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575578.005470:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575578.005474:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575578.005477:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575578.005480:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.005481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575578.005483:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.005485:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:3.0:1788575578.517649:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:3.0:1788575578.517682:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:3.0:1788575578.517689:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575578.517691:0:2352:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:3.0:1788575578.517695:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575578.517699:0:2352:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:3.0:1788575578.517703:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1788575578.517707:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133400.
00000400:00000200:3.0:1788575578.517712:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1788575578.517717:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1788575578.517723:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65b28
00000400:00000010:3.0:1788575578.517725:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65b28.
00000100:00000001:3.0:1788575578.517730:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575578.517740:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575578.517764:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575578.517768:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575578.517773:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575578.517774:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:3.0:1788575578.517782:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1788575578.517784:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133c00.
00000400:00000200:3.0:1788575578.517786:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1788575578.517790:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1788575578.517794:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65d48
00000400:00000010:3.0:1788575578.517796:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65d48.
00000100:00000001:3.0:1788575578.517798:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575578.517803:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.517809:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575578.517813:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1788575578.517814:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575578.517819:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.517820:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1788575578.517827:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575578.517830:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000100:00100000:2.0:1788575578.517831:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575575/real 1788575578]  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:3.0:1788575578.517840:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1788575578.517846:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:3.0:1788575578.517850:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000001:2.0:1788575578.517851:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575578.517855:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000200:3.0:1788575578.517856:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000010:2.0:1788575578.517859:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:3.0:1788575578.517861:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65000
00000400:00000010:3.0:1788575578.517865:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65000.
00000400:00000200:2.0:1788575578.517866:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000100:00000001:3.0:1788575578.517869:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000400:00000010:2.0:1788575578.517870:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575578.517875:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575578.517876:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575578.517882:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575578.517893:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575578.517901:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575578.517908:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575578.517912:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575578.517917:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575578.517923:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575578.517925:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.517929:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575578.517931:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575578.517936:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.517943:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575578.517948:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.517955:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575578.517958:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.517959:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575578.517961:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.517964:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575578.517968:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575578.517972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575578.517978:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575578.517981:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575578.517985:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575578.517995:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575578.518000:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575578.518004:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575575/real 1788575578]  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575578.518020:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575578.518021:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575578.518023:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575578.518026:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575578.518027:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575578.518030:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575578.518033:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575578.518040:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.518046:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575578.518048:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518050:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575578.518051:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575578.518055:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.518062:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575578.518066:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.518072:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575578.518074:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518076:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575578.518077:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518078:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575578.518080:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575578.518083:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575578.518086:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575578.518088:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575578.518092:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575578.518100:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393219712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575578.518104:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575578.518107:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575578.518110:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393219840/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.518117:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575578.518120:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575578.518124:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575578.518128:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575578.518131:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575578.518133:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575578.518135:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575578.518138:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:2.0:1788575578.518141:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575578.518143:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575578.518145:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:2.0:1788575578.518148:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575578.518150:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518151:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575578.518154:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393219712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.518160:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575578.518161:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575578.518162:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575578.518164:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575578.518165:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575578.518166:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575578.518168:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575578.518169:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:2.0:1788575578.518171:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575578.518172:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575578.518173:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:2.0:1788575578.518176:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575578.518177:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518180:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575578.518182:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575578.518183:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575578.518184:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575578.518188:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575575/real 1788575578]  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575578.518196:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575578.518197:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575578.518200:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575578.518202:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575578.518204:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575578.518206:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575578.518209:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575578.518217:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.518230:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575578.518232:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518234:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575578.518235:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575578.518239:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.518245:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575578.518248:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.518254:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575578.518257:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518258:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575578.518259:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518261:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575578.518263:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575578.518265:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575578.518270:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575578.518273:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575578.518277:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575578.518285:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393219968:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575578.518289:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518291:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575578.518294:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393219968/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575578.518301:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575578.518303:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575578.518317:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575578.518320:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575578.518321:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575578.518323:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575578.518325:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575578.518326:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788575578.518329:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575578.518330:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575578.518332:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:2.0:1788575578.518334:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575578.518336:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518337:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575578.518346:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575578.518349:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575578.518351:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575578.518352:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575578.901523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575578.901532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575578.901538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575578.901540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575578.901543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575578.901544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575578.901601:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575578.901605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575578.901608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575579.029472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575579.029478:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575579.029481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575579.029485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575579.029487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575579.029490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575579.029492:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575579.029494:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575579.029495:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575579.925505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575579.925510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575579.925516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575579.925517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575579.925521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575579.925522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575579.925572:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575579.925575:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575579.925577:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575580.053493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575580.053499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575580.053508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575580.053513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575580.053515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575580.053518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575580.053520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575580.053522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575580.053523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575580.565526:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575580.565537:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575580.565540:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575580.565544:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575580.565550:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575580.565558:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12511
00000400:00000200:2.0:1788575580.565567:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575580.565574:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575580.565578:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575580.565581:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575580.565584:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575580.565586:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575580.565588:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575580.565592:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575580.565594:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575580.565597:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575580.565600:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575580.565603:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575580.565605:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575580.565608:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575580.565610:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575580.565612:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575580.565613:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575580.565617:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575580.565620:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575580.565622:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575580.565625:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575580.565628:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575580.565631:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575580.565633:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575580.565635:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575580.565642:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02d80.
00000020:00000040:2.0:1788575580.565645:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575580.565649:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575580.565650:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575580.565654:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575580.565657:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575580.565660:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575580.565667:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:2.0:1788575580.565673:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575580.565682:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575580.565692:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575580.565696:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575580.565701:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009a9ccdb8] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575580.565725:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575580.565728:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575580.565731:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575580.565733:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575580.565734:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575580.565737:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575580.565740:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12511
00000400:00000200:2.0:1788575580.565744:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575580.565748:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575580.565751:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575580.565754:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575580.565755:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575580.565757:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575580.565761:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575580.565761:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575580.565765:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788575580.565766:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575580.565770:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575580.565772:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575580.565775:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:3.0:1788575580.565775:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788575580.565779:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575580.565784:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575580.565788:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575580.565790:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575580.565794:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575580.565795:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575580.565798:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1788575580.565799:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1788575580.565803:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575580.565804:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575580.565808:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575580.565810:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575580.565812:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575580.565817:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:3.0:1788575580.565817:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393220096:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575580.565823:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.565825:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575580.565826:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575580.565830:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575580.565833:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575580.565834:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575580.565836:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1788575580.565840:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575580.565842:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:3.0:1788575580.565843:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575580.565847:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1788575580.565848:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575580.565853:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575580.565854:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575580.565858:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575580.565860:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000010:2.0:1788575580.565864:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03b80.
00000020:00000040:2.0:1788575580.565868:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:3.0:1788575580.565870:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000001:2.0:1788575580.565871:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575580.565873:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575580.565876:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575580.565878:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:3.0:1788575580.565879:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393220096, portal 25
02000000:00000001:2.0:1788575580.565880:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.565883:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000010:2.0:1788575580.565887:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000001:3.0:1788575580.565887:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000001:2.0:1788575580.565893:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575580.565896:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575580.565902:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575580.565912:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1788575580.565918:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1788575580.565918:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575580.565922:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575580.565922:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575580.565928:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393220096, offset 0
00000100:00000040:2.0:1788575580.565930:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037d167ca] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575580.565942:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575580.565943:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575580.565945:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575580.565950:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575580.565952:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575580.565955:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575580.565955:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575580.565961:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:3.0:1788575580.565965:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1788575580.565966:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12511
00000400:00000200:3.0:1788575580.565974:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575580.565978:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575580.565982:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575580.565985:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575580.565989:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330975:330975:256:4294967295] 192.168.202.110@tcp LPNI seq info [330932:330932:8:4294967295]
00000400:00000200:3.0:1788575580.565993:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575580.565999:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575580.566005:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575580.566010:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575580.566010:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1788575580.566017:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575580.566022:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575580.566023:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575580.566027:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575580.566031:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000010:3.0:1788575580.566032:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000020:00000001:2.0:1788575580.566035:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575580.566038:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575580.566040:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575580.566045:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:3.0:1788575580.566045:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1788575580.566050:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.0:1788575580.566052:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1788575580.566054:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575580.566061:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575580.566063:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575580.566065:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1788575580.566068:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:2.0:1788575580.566071:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575580.566074:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000100:3.0:1788575580.566077:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575580.566081:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000020:00000040:2.0:1788575580.566086:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575580.566087:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1788575580.566092:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575580.566097:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575580.566097:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1788575580.566102:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575580.566106:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
02000000:00000001:2.0:1788575580.566107:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:3.0:1788575580.566111:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
02000000:00000001:2.0:1788575580.566112:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575580.566117:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575580.566118:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000008:00100000:2.0:1788575580.566119:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575580.566124:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575580.566125:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575580.566126:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575580.566132:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00000.
00000020:00000040:2.0:1788575580.566136:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575580.566142:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1788575580.566143:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1788575580.566144:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575580.566148:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1788575580.566149:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575580.566150:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575580.566153:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575580.566159:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000100:00000001:3.0:1788575580.566162:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575580.566164:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.566165:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00100000:2.0:1788575580.566173:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575580.566173:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575580.566181:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575580.566186:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307185081
00000100:00000010:3.0:1788575580.566193:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:2.0:1788575580.566197:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575580.566201:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.566202:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:2.0:1788575580.566207:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bf5936f] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575580.566215:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575580.566218:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575580.566222:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575580.566227:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12521)
00000100:00000001:3.0:1788575580.567678:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307185083
00000100:00000010:3.0:1788575580.567689:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575580.567695:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575580.567701:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575580.567704:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.567707:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575580.567716:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575580/real 1788575580]  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575580.567730:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575580.567732:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575580.567735:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575580.567739:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575580.567741:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575580.567745:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575580.567749:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575580.567756:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.567764:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575580.567766:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.567769:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575580.567771:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575580.567775:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.567782:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575580.567788:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.567794:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575580.567797:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.567798:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575580.567800:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.567802:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575580.567804:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575580.567807:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575580.567811:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575580.567813:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575580.567818:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575580.567824:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393220096:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575580.567829:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.567832:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575580.567835:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393220096/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.567842:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575580.567844:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575580.567847:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575580.567851:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575580.567853:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575580.567855:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575580.567858:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575580.567860:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575580.567864:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575580.567866:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575580.567867:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:3.0:1788575580.567870:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575580.567872:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.567874:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.567881:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575580.567883:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.567885:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575580.567888:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575580.567890:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575580.567895:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.567902:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575580.567904:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575580.567906:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575580.567908:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575580.567911:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575580.567914:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220224:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575580.567917:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575580.567921:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575580.567924:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575580.567926:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575580.567930:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575580.567931:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575580.567937:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575580.567940:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575580.567945:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575580.567950:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393220224, portal 10
00000100:00000001:3.0:1788575580.567952:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575580.567953:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:3.0:1788575580.567959:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.567967:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575580.567970:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575580.567973:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575580.567976:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393220224, offset 0
00000400:00000200:3.0:1788575580.567981:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575580.567988:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575580.567995:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575580.567999:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575580.568002:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575580.568005:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575580.568009:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575580.568012:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330976:330976:256:4294967295] 192.168.202.110@tcp LPNI seq info [330933:330933:8:4294967295]
00000400:00000200:3.0:1788575580.568016:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575580.568024:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575580.568029:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575580.568033:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575580.568048:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575580.568051:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575580.568055:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575580.568058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575580.568062:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575580.568064:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575580.568068:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575580.568072:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575580.568076:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575580.568078:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575580.568080:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575580.568083:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568090:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575580.568092:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568102:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.568104:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575580.568108:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575580.568111:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575580.568114:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575580.568116:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307184168
00000100:00000001:3.0:1788575580.568119:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307185081
00000100:00000001:3.0:1788575580.568121:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307184168
00000100:00000001:3.0:1788575580.568123:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307185083
00000100:00000001:3.0:1788575580.568125:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575580.568127:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.568128:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575580.568132:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575580/real 1788575580]  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575580.568138:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575580.568140:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575580.568141:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575580.568143:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575580.568145:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575580.568147:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575580.568150:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575580.568156:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568161:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575580.568162:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568164:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575580.568165:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575580.568168:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568174:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575580.568178:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568184:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575580.568186:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568188:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575580.568189:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568191:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575580.568193:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575580.568195:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575580.568198:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575580.568201:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575580.568204:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575580.568211:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220224:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575580.568214:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575580.568218:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568224:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575580.568225:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575580.568227:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575580.568229:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575580.568231:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575580.568233:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220352:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575580.568236:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575580.568238:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575580.568239:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575580.568241:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575580.568244:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575580.568246:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575580.568250:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575580.568252:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575580.568256:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575580.568259:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393220352, portal 4
00000100:00000001:3.0:1788575580.568260:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575580.568262:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:3.0:1788575580.568268:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568273:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575580.568275:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575580.568278:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575580.568280:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393220352, offset 0
00000400:00000200:3.0:1788575580.568283:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575580.568286:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575580.568290:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575580.568292:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575580.568294:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575580.568297:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575580.568298:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575580.568301:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330977:330977:256:4294967295] 192.168.202.110@tcp LPNI seq info [330934:330934:8:4294967295]
00000400:00000200:3.0:1788575580.568304:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575580.568310:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575580.568314:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575580.568317:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575580.568321:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575580.568323:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575580.568327:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575580.568330:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575580.568333:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575580.568335:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575580.568365:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575580.568369:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575580.568371:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575580.568373:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575580.568375:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575580.568378:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568385:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575580.568388:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568419:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.568420:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575580.568424:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575580.568427:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575580.568430:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575580.568432:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307184315
00000100:00000001:3.0:1788575580.568434:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307185081
00000100:00000001:3.0:1788575580.568436:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307184315
00000100:00000001:3.0:1788575580.568438:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307185083
00000100:00000001:3.0:1788575580.568440:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575580.568442:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.568443:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575580.568448:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575580/real 1788575580]  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575580.568456:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575580.568457:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575580.568459:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575580.568461:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575580.568462:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575580.568465:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575580.568468:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575580.568476:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568482:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575580.568484:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568486:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575580.568487:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575580.568490:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568496:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575580.568499:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568505:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575580.568508:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568509:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575580.568511:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568512:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575580.568514:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575580.568516:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575580.568520:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575580.568522:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575580.568525:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575580.568533:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220352:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575580.568537:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568539:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575580.568543:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393220224/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568549:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575580.568550:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575580.568553:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575580.568555:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575580.568556:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575580.568557:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575580.568559:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575580.568561:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575580.568564:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575580.568565:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575580.568566:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:3.0:1788575580.568568:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575580.568569:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568571:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575580.568573:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393220352/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575580.568578:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575580.568579:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575580.568581:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575580.568583:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575580.568584:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575580.568585:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575580.568586:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575580.568587:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575580.568590:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575580.568591:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575580.568592:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:3.0:1788575580.568594:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575580.568595:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568597:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575580.568600:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575580.568601:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575580.568603:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575580.568605:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575580.949519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575580.949525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575580.949530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575580.949532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575580.949535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575580.949536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575580.949593:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575580.949597:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575580.949600:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575581.077512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575581.077519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575581.077521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575581.077526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575581.077530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575581.077541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575581.077544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575581.077546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575581.077549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575581.525513:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307185083, at 4307185088
00000100:00000010:1.0:1788575581.525528:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000040:0.0:1788575581.525690:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307185081, at 4307185088
00000100:00000010:0.0:1788575581.525696:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575581.973500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575581.973506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575581.973510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575581.973512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575581.973515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575581.973516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575581.973553:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575581.973556:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575581.973558:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575582.101515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575582.101522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575582.101524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575582.101529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575582.101532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575582.101536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575582.101539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575582.101541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575582.101544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575582.997511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575582.997516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575582.997519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575582.997520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575582.997522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575582.997523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575582.997574:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575582.997576:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575582.997579:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575583.125539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575583.125547:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575583.125549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575583.125555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575583.125558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575583.125562:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575583.125564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575583.125567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575583.125569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575584.021531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575584.021537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575584.021541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575584.021543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575584.021546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575584.021547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575584.021601:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575584.021605:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575584.021608:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575584.149496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575584.149502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575584.149503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575584.149507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575584.149509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575584.149512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575584.149514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575584.149516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575584.149518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575585.045513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575585.045519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575585.045523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575585.045526:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575585.045528:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575585.045530:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575585.045532:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575585.045534:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575585.045535:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575585.045536:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575585.045538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575585.045539:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575585.045540:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575585.045541:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575585.045543:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575585.045545:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575585.045546:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575585.045548:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575585.045615:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575585.045619:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575585.045622:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575585.173549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575585.173557:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575585.173559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575585.173564:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575585.173568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575585.173572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.173574:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575585.173577:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.173580:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575585.685516:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575585.685527:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575585.685530:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575585.685534:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575585.685538:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575585.685545:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12516
00000400:00000200:3.0:1788575585.685560:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575585.685566:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575585.685569:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575585.685573:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575585.685576:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575585.685579:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575585.685581:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575585.685585:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575585.685587:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575585.685590:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575585.685593:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575585.685595:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575585.685597:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575585.685600:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575585.685602:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575585.685604:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575585.685606:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575585.685609:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575585.685612:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575585.685615:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575585.685618:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575585.685621:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575585.685624:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575585.685626:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575585.685628:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575585.685637:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2300.
00000020:00000040:3.0:1788575585.685641:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575585.685645:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575585.685646:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575585.685649:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575585.685651:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575585.685655:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575585.685663:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575585.685669:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575585.685678:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575585.685689:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575585.685692:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575585.685697:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc4bce75] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575585.685719:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575585.685722:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575585.685725:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575585.685727:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575585.685735:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575585.685739:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575585.685739:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:2.0:1788575585.685743:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1788575585.685746:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12516
00000100:00000001:2.0:1788575585.685747:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575585.685754:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575585.685754:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575585.685762:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1788575585.685766:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1788575585.685767:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575585.685768:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575585.685775:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1788575585.685775:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575585.685778:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575585.685780:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575585.685783:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575585.685784:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575585.685788:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00100000:2.0:1788575585.685790:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393220480:192.168.202.110@tcp:250:kworker.0
00000020:00000001:3.0:1788575585.685792:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575585.685794:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575585.685798:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:3.0:1788575585.685798:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575585.685803:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
02000000:00000001:2.0:1788575585.685804:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:3.0:1788575585.685806:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575585.685808:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1788575585.685810:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575585.685813:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575585.685820:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
00000100:00000040:3.0:1788575585.685823:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575585.685824:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575585.685829:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575585.685832:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575585.685834:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575585.685836:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:2.0:1788575585.685841:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000020:00000040:3.0:1788575585.685841:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575585.685848:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:2.0:1788575585.685850:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000001:3.0:1788575585.685851:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575585.685855:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:2.0:1788575585.685858:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393220480, portal 25
02000000:00000001:3.0:1788575585.685860:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575585.685863:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1788575585.685865:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.685866:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000001:3.0:1788575585.685870:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1788575585.685876:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1788575585.685876:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575585.685883:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575585.685887:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.685894:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:3.0:1788575585.685895:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0a80.
00000100:00000040:2.0:1788575585.685899:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575585.685900:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:2.0:1788575585.685904:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
02000000:00000001:3.0:1788575585.685905:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575585.685908:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575585.685910:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393220480, offset 0
00000100:00000001:3.0:1788575585.685912:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575585.685915:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575585.685917:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575585.685920:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000010:3.0:1788575585.685924:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575585.685929:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575585.685931:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1788575585.685938:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575585.685941:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575585.685950:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788575585.685952:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575585.685956:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575585.685957:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575585.685964:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b298c26f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575585.685965:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575585.685971:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575585.685978:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330978:330978:256:4294967295] 192.168.202.110@tcp LPNI seq info [330935:330935:8:4294967295]
00000100:00000001:3.0:1788575585.685984:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575585.685988:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575585.685989:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575585.685995:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575585.685999:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575585.686002:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575585.686003:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1788575585.686008:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:2.0:1788575585.686015:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575585.686015:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12516
00000400:00000200:3.0:1788575585.686023:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:2.0:1788575585.686024:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000400:00000200:3.0:1788575585.686029:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:2.0:1788575585.686034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1788575585.686034:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575585.686041:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.0:1788575585.686042:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:3.0:1788575585.686045:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575585.686050:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575585.686053:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575585.686054:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.1:1788575585.686058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:3.0:1788575585.686059:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575585.686061:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575585.686065:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000100:2.0:1788575585.686069:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575585.686069:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000010:2.0:1788575585.686073:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000020:00000001:3.0:1788575585.686074:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575585.686079:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575585.686081:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1788575585.686084:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575585.686089:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575585.686090:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1788575585.686095:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575585.686099:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000100:00000001:3.0:1788575585.686099:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:2.0:1788575585.686103:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000020:00000040:3.0:1788575585.686105:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788575585.686110:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00080000:3.0:1788575585.686111:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575585.686114:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575585.686117:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:2.0:1788575585.686118:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575585.686121:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575585.686125:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575585.686129:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575585.686132:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575585.686135:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686137:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1788575585.686137:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575585.686143:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000100:00000040:2.0:1788575585.686144:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575585.686146:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575585.686150:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575585.686152:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575585.686155:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575585.686157:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575585.686160:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686163:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575585.686164:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:2.0:1788575585.686167:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575585.686173:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575585.686176:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575585.686180:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1788575585.686183:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575585.686185:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307190201
00000100:00000001:3.0:1788575585.686193:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575585.686195:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575585.686196:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c3c0.
00000100:00000040:3.0:1788575585.686197:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575585.686205:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686206:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:3.0:1788575585.686207:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575585.686211:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575585.686215:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12526)
00000100:00000001:2.0:1788575585.686221:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307190201
00000100:00000010:2.0:1788575585.686227:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cc00.
00000100:00000001:2.0:1788575585.686231:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575585.686235:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575585.686237:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686239:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575585.686246:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575585/real 1788575585]  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575585.686263:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575585.686265:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575585.686267:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575585.686270:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575585.686272:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575585.686274:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575585.686278:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575585.686286:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686293:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575585.686295:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686297:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575585.686299:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575585.686303:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686310:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575585.686315:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686322:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575585.686325:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686327:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575585.686328:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686330:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575585.686333:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575585.686336:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575585.686339:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575585.686342:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575585.686346:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575585.686355:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393220480:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575585.686359:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575585.686377:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393220480/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686384:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575585.686387:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575585.686389:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575585.686412:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575585.686414:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575585.686415:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575585.686431:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575585.686433:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575585.686437:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575585.686438:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575585.686440:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:2.0:1788575585.686443:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575585.686444:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686446:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686453:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575585.686455:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686457:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575585.686458:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575585.686459:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575585.686464:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686470:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575585.686471:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575585.686474:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575585.686475:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575585.686477:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575585.686480:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575585.686484:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575585.686485:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575585.686487:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575585.686488:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575585.686491:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575585.686493:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575585.686497:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575585.686500:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575585.686503:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575585.686506:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393220608, portal 10
00000100:00000001:2.0:1788575585.686508:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575585.686509:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:2.0:1788575585.686515:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686532:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575585.686534:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575585.686537:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575585.686540:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393220608, offset 0
00000400:00000200:2.0:1788575585.686544:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575585.686548:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575585.686552:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575585.686555:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575585.686557:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575585.686560:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575585.686562:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575585.686564:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330979:330979:256:4294967295] 192.168.202.110@tcp LPNI seq info [330936:330936:8:4294967295]
00000400:00000200:2.0:1788575585.686568:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575585.686573:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575585.686575:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575585.686578:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575585.686580:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575585.686582:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575585.686585:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575585.686587:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575585.686589:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575585.686590:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575585.686592:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575585.686594:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575585.686596:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575585.686598:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575585.686599:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575585.686602:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686606:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575585.686608:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686612:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686613:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575585.686615:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575585.686617:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575585.686619:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575585.686621:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307189287
00000100:00000001:2.0:1788575585.686622:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307190201
00000100:00000001:2.0:1788575585.686624:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307189287
00000100:00000001:2.0:1788575585.686625:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307190201
00000100:00000001:2.0:1788575585.686626:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575585.686627:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686628:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575585.686630:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575585/real 1788575585]  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575585.686635:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575585.686636:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575585.686637:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575585.686638:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575585.686639:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575585.686641:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575585.686642:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575585.686647:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686651:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575585.686652:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686653:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575585.686653:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575585.686655:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686659:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575585.686661:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686664:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575585.686666:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686667:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575585.686667:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686668:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575585.686670:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575585.686671:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575585.686673:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575585.686674:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575585.686676:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575585.686680:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575585.686683:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575585.686685:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686689:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575585.686689:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575585.686691:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575585.686692:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575585.686693:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575585.686694:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575585.686696:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575585.686697:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575585.686698:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575585.686699:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575585.686701:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575585.686702:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575585.686705:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575585.686706:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575585.686708:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575585.686710:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393220736, portal 4
00000100:00000001:2.0:1788575585.686711:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575585.686712:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575585.686715:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686719:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575585.686721:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575585.686722:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575585.686723:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393220736, offset 0
00000400:00000200:2.0:1788575585.686725:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575585.686727:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575585.686729:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575585.686731:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575585.686732:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575585.686734:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575585.686735:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575585.686737:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330980:330980:256:4294967295] 192.168.202.110@tcp LPNI seq info [330937:330937:8:4294967295]
00000400:00000200:2.0:1788575585.686739:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575585.686743:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575585.686745:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575585.686746:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575585.686749:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575585.686750:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575585.686754:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575585.686756:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575585.686760:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575585.686762:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575585.686764:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575585.686767:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575585.686770:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575585.686772:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575585.686773:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575585.686777:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686784:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575585.686787:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686793:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686794:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575585.686797:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575585.686800:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575585.686803:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575585.686805:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307189433
00000100:00000001:2.0:1788575585.686807:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307190201
00000100:00000001:2.0:1788575585.686809:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307189433
00000100:00000001:2.0:1788575585.686812:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307190201
00000100:00000001:2.0:1788575585.686814:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575585.686815:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686817:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575585.686821:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575585/real 1788575585]  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575585.686826:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575585.686827:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575585.686828:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575585.686829:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575585.686831:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575585.686832:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575585.686834:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575585.686838:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686842:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575585.686842:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686843:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575585.686844:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575585.686846:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686850:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575585.686852:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686855:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575585.686857:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686858:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575585.686858:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686860:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575585.686861:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575585.686862:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575585.686864:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575585.686866:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575585.686868:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575585.686872:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575585.686875:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686876:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575585.686877:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393220608/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686881:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575585.686882:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575585.686883:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575585.686885:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575585.686886:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575585.686887:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575585.686888:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575585.686889:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575585.686890:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575585.686891:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575585.686892:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:2.0:1788575585.686894:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575585.686895:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686896:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575585.686897:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393220736/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575585.686900:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575585.686901:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575585.686902:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575585.686903:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575585.686904:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575585.686905:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575585.686906:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575585.686907:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575585.686908:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575585.686909:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575585.686909:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575585.686911:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575585.686911:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686912:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575585.686914:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575585.686915:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575585.686916:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575585.686917:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575586.069508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575586.069514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575586.069520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575586.069521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575586.069526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575586.069527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575586.069570:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575586.069572:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575586.069575:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575586.197532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575586.197539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575586.197540:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575586.197543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575586.197546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575586.197550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575586.197552:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575586.197554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575586.197556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575586.645498:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307190201, at 4307190208
00000100:00000040:1.0:1788575586.645498:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307190201, at 4307190208
00000100:00000010:0.0:1788575586.645513:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000100:00000010:1.0:1788575586.645515:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00010000:00000001:3.0:1788575587.093516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575587.093523:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575587.093526:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575587.093529:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575587.093534:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575587.093536:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575587.093538:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575587.093540:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575587.093543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575587.093544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575587.093547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575587.093549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575587.093614:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575587.093618:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575587.093621:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575587.221514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575587.221522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575587.221524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575587.221530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575587.221534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575587.221538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575587.221542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575587.221544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575587.221547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575588.117527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575588.117534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575588.117539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575588.117541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575588.117543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575588.117544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575588.117604:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575588.117608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575588.117610:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575588.245547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575588.245555:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575588.245557:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575588.245571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575588.245575:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575588.245579:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575588.245581:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575588.245584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575588.245587:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575589.141516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575589.141520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575589.141524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575589.141525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575589.141527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575589.141528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575589.141579:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575589.141582:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575589.141584:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575589.269521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575589.269529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575589.269531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575589.269537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575589.269540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575589.269544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575589.269547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575589.269549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575589.269552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575590.165466:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575590.165471:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575590.165475:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575590.165476:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575590.165478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575590.165479:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575590.165518:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575590.165522:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575590.165526:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575590.293509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575590.293515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575590.293517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575590.293522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575590.293525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575590.293529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575590.293539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575590.293542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575590.293546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575590.805531:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575590.805541:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575590.805544:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575590.805548:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575590.805552:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575590.805558:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12521
00000400:00000200:1.0:1788575590.805568:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575590.805573:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575590.805577:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575590.805580:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575590.805583:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575590.805585:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575590.805588:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575590.805591:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575590.805594:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575590.805597:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575590.805600:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575590.805603:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575590.805605:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575590.805608:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575590.805610:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575590.805612:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575590.805614:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575590.805617:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575590.805621:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575590.805623:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575590.805626:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575590.805628:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575590.805631:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575590.805633:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575590.805635:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575590.805643:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788575590.805646:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575590.805650:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575590.805651:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575590.805654:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575590.805656:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575590.805660:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575590.805666:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788575590.805671:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575590.805680:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575590.805690:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575590.805694:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575590.805699:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575590.805726:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575590.805729:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575590.805732:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575590.805734:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575590.805735:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575590.805738:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575590.805740:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12521
00000400:00000200:1.0:1788575590.805744:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575590.805748:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575590.805750:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575590.805754:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575590.805756:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575590.805757:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788575590.805759:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575590.805763:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575590.805763:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575590.805768:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1788575590.805769:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575590.805773:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575590.805775:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575590.805779:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000040:2.0:1788575590.805779:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788575590.805785:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575590.805790:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575590.805792:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575590.805794:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575590.805798:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575590.805799:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575590.805801:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788575590.805804:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575590.805805:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575590.805810:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00100000:2.0:1788575590.805811:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393220864:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788575590.805816:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575590.805817:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:1.0:1788575590.805821:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575590.805821:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575590.805825:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575590.805827:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575590.805828:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788575590.805832:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575590.805834:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:1.0:1788575590.805835:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575590.805837:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575590.805843:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575590.805843:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575590.805849:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575590.805849:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00000001:1.0:1788575590.805853:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:2.0:1788575590.805856:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000008:00100000:1.0:1788575590.805858:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575590.805862:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575590.805863:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393220864, portal 25
00000100:00000001:1.0:1788575590.805865:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.805867:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575590.805869:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345471232 : -130483364080384 : ffff8953796f5500)
00000100:00000010:1.0:1788575590.805871:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000100:00000040:2.0:1788575590.805874:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788575590.805876:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575590.805881:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575590.805883:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.805885:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1788575590.805887:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:2.0:1788575590.805889:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788575590.805890:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1788575590.805893:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
02000000:00000001:1.0:1788575590.805894:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575590.805897:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393220864, offset 0
02000000:00000010:1.0:1788575590.805901:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000400:00000200:2.0:1788575590.805903:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575590.805908:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575590.805912:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:1.0:1788575590.805917:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575590.805922:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575590.805928:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575590.805932:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:1.0:1788575590.805933:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575590.805937:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1788575590.805939:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575590.805941:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575590.805949:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330981:330981:256:4294967295] 192.168.202.110@tcp LPNI seq info [330938:330938:8:4294967295]
00000100:00000040:1.0:1788575590.805950:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575590.805953:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575590.805965:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575590.805965:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:1.0:1788575590.805971:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1788575590.805972:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788575590.805978:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000010:2.0:1788575590.805978:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000100:00000001:1.0:1788575590.805981:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575590.805984:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:2.0:1788575590.805987:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788575590.805989:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575590.805993:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12521
00000400:00000200:1.0:1788575590.805996:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575590.806000:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:2.0:1788575590.806001:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00080000:1.0:1788575590.806004:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:2.1:1788575590.806008:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788575590.806010:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575590.806012:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:1.0:1788575590.806014:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000100:2.0:1788575590.806018:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788575590.806019:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575590.806021:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000100:00000001:1.0:1788575590.806025:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575590.806028:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:1.0:1788575590.806029:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575590.806032:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575590.806036:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:2.0:1788575590.806036:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:1.0:1788575590.806040:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575590.806043:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575590.806044:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000020:00000001:1.0:1788575590.806046:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:2.0:1788575590.806049:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000040:1.0:1788575590.806052:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575590.806055:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575590.806057:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:1.0:1788575590.806058:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575590.806060:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575590.806062:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575590.806065:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575590.806068:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575590.806071:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:2.0:1788575590.806073:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788575590.806074:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575590.806080:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575590.806084:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575590.806087:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575590.806091:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806092:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1788575590.806093:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575590.806097:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575615 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1788575590.806098:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788575590.806103:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575590.806106:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575590.806107:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575590.806108:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575590.806110:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575590.806113:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575590.806118:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:2.0:1788575590.806120:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575590.806124:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806124:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575590.806131:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575590.806132:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575590.806141:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575590.806147:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575590.806147:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307195321
00000100:00000001:1.0:1788575590.806149:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575590.806156:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1788575590.806156:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cc00.
00000100:00000001:1.0:1788575590.806165:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806165:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:1.0:1788575590.806168:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575590.806171:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575590.806175:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12532)
00000100:00000001:2.0:1788575590.806177:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307195321
00000100:00000010:2.0:1788575590.806181:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c3c0.
00000100:00000001:2.0:1788575590.806183:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575590.806187:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575590.806188:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806191:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575590.806196:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575590/real 1788575590]  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575590.806205:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575590.806206:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575590.806209:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575590.806212:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575590.806214:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575590.806217:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575590.806221:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575590.806229:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806236:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575590.806238:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806240:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575590.806242:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575590.806247:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806251:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575590.806254:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806258:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575590.806260:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806261:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575590.806261:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806263:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575590.806264:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575590.806266:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575590.806269:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575590.806270:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575590.806273:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575590.806278:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393220864:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575590.806281:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806282:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575590.806283:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5500 x1875444393220864/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806287:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575590.806289:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575590.806291:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575590.806293:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575590.806294:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575590.806295:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575590.806296:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575590.806298:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:2.0:1788575590.806300:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575590.806301:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575590.806303:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5500.
00000100:00000001:2.0:1788575590.806304:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575590.806305:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806307:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806312:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575590.806313:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806314:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575590.806315:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575590.806316:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575590.806318:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806322:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575590.806323:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575590.806325:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575590.806326:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575590.806327:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575590.806329:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220992:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575590.806331:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575590.806332:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575590.806333:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575590.806334:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575590.806336:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575590.806337:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575590.806340:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575590.806341:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575590.806344:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575590.806346:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393220992, portal 10
00000100:00000001:2.0:1788575590.806347:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575590.806348:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345469440 : -130483364082176 : ffff8953796f4e00)
00000100:00000040:2.0:1788575590.806352:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806356:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575590.806357:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575590.806359:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575590.806360:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393220992, offset 0
00000400:00000200:2.0:1788575590.806363:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575590.806365:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575590.806367:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575590.806369:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575590.806371:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575590.806372:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575590.806374:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575590.806376:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330982:330982:256:4294967295] 192.168.202.110@tcp LPNI seq info [330939:330939:8:4294967295]
00000400:00000200:2.0:1788575590.806378:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575590.806383:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575590.806386:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575590.806388:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575590.806390:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575590.806419:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788575590.806422:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575590.806424:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:2.0:1788575590.806427:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575590.806428:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575590.806430:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575590.806432:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575590.806434:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575590.806435:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575590.806437:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575590.806439:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806443:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575590.806445:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575615 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806449:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806450:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575590.806452:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575590.806454:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575590.806468:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575590.806470:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307194407
00000100:00000001:2.0:1788575590.806472:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307195321
00000100:00000001:2.0:1788575590.806474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307194407
00000100:00000001:2.0:1788575590.806475:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307195321
00000100:00000001:2.0:1788575590.806476:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575590.806477:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806479:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575590.806481:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575590/real 1788575590]  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575590.806486:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575590.806487:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575590.806488:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575590.806490:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575590.806491:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575590.806492:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575590.806498:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575590.806503:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806507:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575590.806508:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806509:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575590.806509:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575590.806511:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806515:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575590.806517:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806521:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575590.806522:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806523:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575590.806524:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806525:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575590.806526:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575590.806528:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575590.806530:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575590.806531:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575590.806533:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575590.806538:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393220992:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575590.806540:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575590.806542:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806545:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575590.806549:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575590.806550:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575590.806551:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575590.806552:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575590.806554:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221120:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575590.806556:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575590.806557:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575590.806558:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575590.806559:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575590.806562:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575590.806563:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575590.806565:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575590.806567:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575590.806568:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575590.806571:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393221120, portal 4
00000100:00000001:2.0:1788575590.806572:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575590.806573:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345481088 : -130483364070528 : ffff8953796f7b80)
00000100:00000040:2.0:1788575590.806576:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806580:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575590.806581:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575590.806583:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575590.806584:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393221120, offset 0
00000400:00000200:2.0:1788575590.806587:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575590.806589:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575590.806591:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575590.806593:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575590.806594:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575590.806596:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575590.806597:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575590.806599:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330983:330983:256:4294967295] 192.168.202.110@tcp LPNI seq info [330940:330940:8:4294967295]
00000400:00000200:2.0:1788575590.806602:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575590.806605:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575590.806608:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575590.806609:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575590.806612:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575590.806613:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1788575590.806616:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575590.806617:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:2.0:1788575590.806619:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575590.806621:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575590.806622:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575590.806624:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575590.806626:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575590.806627:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575590.806628:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575590.806630:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806634:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575590.806636:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575615 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806640:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806641:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575590.806642:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575590.806644:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575590.806646:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575590.806648:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307194553
00000100:00000001:2.0:1788575590.806649:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307195321
00000100:00000001:2.0:1788575590.806651:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307194553
00000100:00000001:2.0:1788575590.806652:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307195321
00000100:00000001:2.0:1788575590.806653:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575590.806654:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806655:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575590.806657:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575590/real 1788575590]  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575590.806661:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575590.806661:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575590.806662:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575590.806664:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575590.806665:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575590.806666:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575590.806667:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575590.806672:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806675:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575590.806676:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806677:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575590.806677:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575590.806679:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806683:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575590.806685:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806688:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575590.806690:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806691:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575590.806692:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806693:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575590.806694:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575590.806695:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575590.806697:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575590.806698:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575590.806700:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575590.806704:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221120:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575590.806719:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806720:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575590.806722:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4e00 x1875444393220992/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806726:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575590.806727:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575590.806728:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575590.806729:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575590.806730:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575590.806731:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575590.806732:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575590.806734:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:2.0:1788575590.806736:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575590.806737:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575590.806738:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4e00.
00000100:00000001:2.0:1788575590.806740:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575590.806741:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806742:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575590.806744:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7b80 x1875444393221120/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575615 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575590.806752:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575590.806752:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575590.806754:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575590.806755:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575590.806755:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575590.806756:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575590.806757:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575590.806758:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:2.0:1788575590.806759:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575590.806759:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575590.806760:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7b80.
00000100:00000001:2.0:1788575590.806761:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575590.806762:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806763:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575590.806765:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575590.806766:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575590.806767:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575590.806768:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575591.189494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575591.189501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575591.189506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575591.189508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575591.189511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575591.189513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575591.189565:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575591.189567:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575591.189570:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575591.317499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575591.317514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575591.317521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575591.317526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575591.317533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575591.317537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575591.317540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575591.317542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575591.317545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575591.765507:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307195321, at 4307195328
00000100:00000040:1.0:1788575591.765507:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307195321, at 4307195328
00000100:00000010:1.0:1788575591.765523:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000100:00000010:0.0:1788575591.765526:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00010000:00000001:3.0:1788575592.213497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575592.213502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575592.213505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575592.213506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575592.213508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575592.213509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575592.213566:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575592.213569:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575592.213572:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575592.341516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575592.341523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575592.341525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575592.341530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575592.341534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575592.341537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575592.341540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575592.341542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575592.341545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575593.237479:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575593.237486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575593.237492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575593.237493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575593.237497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575593.237499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575593.237532:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575593.237535:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575593.237539:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575593.365488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575593.365495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575593.365497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575593.365502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575593.365505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575593.365509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575593.365511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575593.365514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575593.365516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575594.261536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575594.261545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575594.261551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575594.261552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575594.261557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575594.261558:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575594.261616:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575594.261621:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575594.261624:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575594.389490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575594.389495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575594.389497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575594.389501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575594.389505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575594.389507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575594.389509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575594.389512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575594.389515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575595.285505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575595.285510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575595.285513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575595.285516:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575595.285518:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575595.285520:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575595.285523:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575595.285524:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575595.285525:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575595.285527:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575595.285529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575595.285530:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575595.285531:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575595.285532:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575595.285533:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575595.285534:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575595.285535:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575595.285536:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575595.285593:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575595.285596:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575595.285598:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575595.413510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575595.413517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575595.413519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575595.413525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575595.413528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575595.413532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575595.413535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575595.413542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575595.413544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575595.925484:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575595.925495:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000800:00000200:3.0:1788575595.925497:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000100:00000001:2.0:1788575595.925499:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:3.0:1788575595.925504:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000100:00080000:2.0:1788575595.925505:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000800:00000200:3.0:1788575595.925509:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1788575595.925511:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575595.925523:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12527
00000400:00000200:2.0:1788575595.925535:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575595.925544:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575595.925547:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575595.925551:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925555:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575595.925558:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925560:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575595.925563:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575595.925565:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575595.925567:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575595.925570:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575595.925571:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575595.925573:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575595.925575:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925577:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575595.925578:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925579:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575595.925581:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575595.925584:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925585:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575595.925587:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575595.925590:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575595.925592:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575595.925593:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575595.925595:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575595.925601:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01880.
00000020:00000040:2.0:1788575595.925603:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575595.925606:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575595.925608:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575595.925610:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575595.925612:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575595.925615:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575595.925620:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12c00.
00000100:00000001:2.0:1788575595.925626:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575595.925633:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575595.925640:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575595.925644:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575595.925647:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ab28ae01] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000001:1.0:1788575595.925660:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000001:2.0:1788575595.925663:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575595.925667:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575595.925669:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575595.925671:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575595.925672:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575595.925674:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:0.0:1788575595.925675:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00080000:2.0:1788575595.925677:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12527
00000800:00000001:1.0:1788575595.925678:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1788575595.925685:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000001:1.0:1788575595.925686:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000400:00000200:2.0:1788575595.925689:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000001:1.0:1788575595.925690:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1788575595.925692:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575595.925695:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925697:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575595.925699:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925701:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575595.925703:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575595.925703:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575595.925705:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575595.925707:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575595.925708:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575595.925709:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575595.925712:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575595.925712:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575595.925715:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575595.925716:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1788575595.925718:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575595.925720:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788575595.925721:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575595.925724:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:3.0:1788575595.925727:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575595.925728:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925733:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575595.925736:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575595.925741:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1788575595.925742:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575595.925742:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575595.925744:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575595.925746:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575595.925749:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575595.925754:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575595.925755:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575595.925755:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575595.925757:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:2.0:1788575595.925759:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00380.
02000000:00000001:3.0:1788575595.925760:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575595.925762:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575595.925766:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:3.0:1788575595.925766:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393221248:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1788575595.925769:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575595.925773:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1788575595.925773:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575595.925774:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575595.925777:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575595.925778:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:2.0:1788575595.925780:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575595.925780:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575595.925785:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788575595.925786:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575595.925790:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
00000100:00100000:2.0:1788575595.925791:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1788575595.925793:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575595.925801:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575595.925801:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575595.925805:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575595.925806:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000040:2.0:1788575595.925809:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fc98b80] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788575595.925813:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575595.925817:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575595.925819:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1788575595.925820:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393221248, portal 25
00000100:00080000:2.0:1788575595.925822:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575595.925825:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1788575595.925825:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1788575595.925828:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575595.925828:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530777728 : -130484178773888 : ffff895348e01880)
00000100:00000001:2.0:1788575595.925833:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575595.925838:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12527
00000100:00000040:3.0:1788575595.925838:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575595.925845:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575595.925850:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575595.925854:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575595.925854:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575595.925859:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575595.925860:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925864:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1788575595.925864:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000040:2.0:1788575595.925868:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:3.0:1788575595.925869:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393221248, offset 0
00000100:00000001:2.0:1788575595.925870:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575595.925873:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575595.925874:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575595.925874:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575595.925875:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575595.925876:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575595.925878:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575595.925882:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575595.925882:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925887:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575595.925888:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925889:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575595.925891:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575595.925893:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575595.925894:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575595.925898:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575595.925899:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575595.925900:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575595.925902:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:2.0:1788575595.925903:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575595.925907:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575595.925909:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00100000:2.0:1788575595.925910:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:3.0:1788575595.925913:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00000001:2.0:1788575595.925916:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575595.925918:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1788575595.925922:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575595.925926:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000010:2.0:1788575595.925928:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00000.
00000400:00000200:3.0:1788575595.925929:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330984:330984:256:4294967295] 192.168.202.110@tcp LPNI seq info [330941:330941:8:4294967295]
00000020:00000040:2.0:1788575595.925934:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575595.925935:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575595.925941:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575595.925943:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575595.925947:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1788575595.925948:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1788575595.925952:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.0:1788575595.925955:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788575595.925956:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575595.925959:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
02000000:00000010:2.0:1788575595.925963:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000800:00000200:3.0:1788575595.925963:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575595.925970:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575595.925970:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788575595.925975:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:2.0:1788575595.925978:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1788575595.925979:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1788575595.925989:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575595.925992:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000100:00000001:2.0:1788575595.925993:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575595.925996:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575595.925997:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1788575595.926001:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bf5936f] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575595.926004:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575595.926011:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000100:00000001:2.0:1788575595.926012:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575595.926014:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000040:2.0:1788575595.926016:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575595.926018:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:2.0:1788575595.926020:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000200:3.0:1788575595.926024:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575595.926026:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12537)
00000100:00000001:3.0:1788575595.926034:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575595.926036:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575620 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926041:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575595.926043:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575595.926047:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575595.926054:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575595.926056:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307200441
00000100:00000010:3.0:1788575595.926060:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575595.926066:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575595.926070:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307200441
00000100:00000010:3.0:1788575595.926072:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575595.926074:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575595.926076:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575595.926078:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575595.926079:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575595.926082:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575595/real 1788575595]  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575595.926088:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575595.926090:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575595.926091:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575595.926093:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575595.926094:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575595.926095:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575595.926097:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575595.926103:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926106:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575595.926108:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926109:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575595.926111:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575595.926114:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926118:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575595.926120:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926126:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575595.926128:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926130:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575595.926131:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926134:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575595.926135:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575595.926139:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575595.926142:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575595.926144:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575595.926149:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575595.926162:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393221248:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575595.926167:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926169:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575595.926173:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01880 x1875444393221248/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926179:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575595.926181:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575595.926184:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575595.926187:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575595.926189:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575595.926190:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575595.926192:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575595.926195:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575595.926198:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575595.926199:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575595.926201:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01880.
00000100:00000001:3.0:1788575595.926204:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575595.926205:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926207:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926214:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575595.926215:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575595.926217:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575595.926219:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575595.926220:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575595.926225:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926231:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575595.926232:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575595.926234:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575595.926236:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575595.926238:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575595.926241:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221376:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575595.926244:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575595.926246:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575595.926248:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575595.926249:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575595.926252:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575595.926254:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575595.926257:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575595.926260:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575595.926263:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575595.926266:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393221376, portal 10
00000100:00000001:3.0:1788575595.926268:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575595.926270:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:3.0:1788575595.926275:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926282:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575595.926284:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575595.926286:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575595.926288:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393221376, offset 0
00000400:00000200:3.0:1788575595.926292:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575595.926296:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575595.926299:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575595.926302:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575595.926304:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575595.926306:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575595.926317:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575595.926320:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330985:330985:256:4294967295] 192.168.202.110@tcp LPNI seq info [330942:330942:8:4294967295]
00000400:00000200:3.0:1788575595.926324:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575595.926331:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575595.926335:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575595.926339:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575595.926343:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575595.926345:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788575595.926349:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575595.926352:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1788575595.926355:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575595.926357:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575595.926360:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575595.926363:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575595.926366:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575595.926368:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575595.926370:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575595.926374:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926382:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575595.926385:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575620 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926414:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575595.926415:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575595.926419:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575595.926422:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575595.926425:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575595.926427:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307199527
00000100:00000001:3.0:1788575595.926429:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307200441
00000100:00000001:3.0:1788575595.926431:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307199527
00000100:00000001:3.0:1788575595.926432:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307200441
00000100:00000001:3.0:1788575595.926434:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575595.926435:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575595.926436:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575595.926439:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575595/real 1788575595]  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575595.926445:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575595.926445:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575595.926446:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575595.926448:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575595.926449:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575595.926450:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575595.926452:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575595.926457:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926461:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575595.926462:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926463:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575595.926463:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575595.926465:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926469:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575595.926471:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926486:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575595.926488:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926489:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575595.926490:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926491:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575595.926492:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575595.926494:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575595.926497:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575595.926498:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575595.926500:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575595.926505:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221376:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575595.926507:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575595.926509:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926513:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575595.926514:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575595.926515:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575595.926516:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575595.926517:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575595.926519:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575595.926520:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575595.926522:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575595.926523:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575595.926524:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575595.926526:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575595.926527:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575595.926530:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575595.926531:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575595.926534:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575595.926535:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393221504, portal 4
00000100:00000001:3.0:1788575595.926536:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575595.926537:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:3.0:1788575595.926542:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926554:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575595.926556:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575595.926558:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575595.926561:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393221504, offset 0
00000400:00000200:3.0:1788575595.926565:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575595.926568:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575595.926572:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575595.926574:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575595.926577:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575595.926579:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575595.926581:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575595.926584:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330986:330986:256:4294967295] 192.168.202.110@tcp LPNI seq info [330943:330943:8:4294967295]
00000400:00000200:3.0:1788575595.926587:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575595.926593:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575595.926596:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575595.926598:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575595.926600:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575595.926602:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788575595.926612:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575595.926613:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1788575595.926616:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575595.926617:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575595.926619:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575595.926621:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575595.926623:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575595.926624:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575595.926626:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575595.926628:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926633:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575595.926635:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575620 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926643:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575595.926644:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575595.926646:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575595.926648:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575595.926650:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575595.926651:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307199673
00000100:00000001:3.0:1788575595.926652:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307200441
00000100:00000001:3.0:1788575595.926654:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307199673
00000100:00000001:3.0:1788575595.926655:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307200441
00000100:00000001:3.0:1788575595.926656:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575595.926657:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575595.926700:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575595.926703:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575595/real 1788575595]  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575595.926708:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575595.926709:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575595.926711:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575595.926712:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575595.926713:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575595.926715:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575595.926716:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575595.926721:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926725:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575595.926726:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926727:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575595.926727:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575595.926729:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926734:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575595.926736:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926740:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575595.926741:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926742:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575595.926743:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926744:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575595.926745:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575595.926746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575595.926748:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575595.926749:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575595.926751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575595.926755:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575595.926757:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926758:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575595.926760:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393221376/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926764:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575595.926765:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575595.926766:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575595.926767:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575595.926768:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575595.926769:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575595.926770:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575595.926771:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575595.926773:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575595.926774:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575595.926775:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:3.0:1788575595.926776:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575595.926777:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926778:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575595.926780:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393221504/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575620 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575595.926784:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575595.926784:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575595.926785:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575595.926786:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575595.926787:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575595.926787:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575595.926788:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575595.926789:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575595.926790:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575595.926791:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575595.926791:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:3.0:1788575595.926792:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575595.926793:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926794:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575595.926796:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575595.926797:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575595.926798:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575595.926799:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575596.309532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575596.309539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575596.309544:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575596.309546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575596.309549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575596.309551:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575596.309614:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575596.309617:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575596.309620:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575596.437546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575596.437553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575596.437556:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575596.437561:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575596.437564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575596.437568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575596.437570:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575596.437573:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575596.437576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575596.885498:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307200441, at 4307200448
00000100:00000040:1.0:1788575596.885498:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307200441, at 4307200448
00000100:00000010:0.0:1788575596.885511:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:1.0:1788575596.885513:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00010000:00000001:3.0:1788575597.333540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575597.333549:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575597.333551:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575597.333554:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575597.333558:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575597.333567:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575597.333569:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575597.333571:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575597.333575:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575597.333576:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575597.333580:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575597.333584:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575597.333633:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575597.333637:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575597.333644:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575597.461457:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575597.461465:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575597.461467:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575597.461472:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575597.461476:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575597.461489:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575597.461493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575597.461495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575597.461498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575598.357518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575598.357524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575598.357528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575598.357529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575598.357531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575598.357532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575598.357575:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575598.357578:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575598.357581:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575598.485519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575598.485526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575598.485529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575598.485534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575598.485538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575598.485542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575598.485544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575598.485547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575598.485550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575599.381548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575599.381555:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575599.381561:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575599.381563:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575599.381566:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575599.381567:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575599.381634:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575599.381638:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575599.381641:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575599.509523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575599.509530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575599.509532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575599.509538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575599.509541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575599.509545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575599.509547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575599.509550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575599.509553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575600.405531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575600.405536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575600.405542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575600.405543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575600.405547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575600.405548:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575600.405609:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575600.405613:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575600.405617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575600.533456:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575600.533461:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575600.533462:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575600.533466:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575600.533468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575600.533471:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575600.533473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575600.533475:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575600.533477:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575601.045504:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575601.045513:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575601.045515:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575601.045519:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575601.045522:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575601.045528:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12532
00000400:00000200:2.0:1788575601.045537:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575601.045543:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575601.045545:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575601.045549:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045551:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575601.045554:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045556:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575601.045560:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575601.045562:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575601.045565:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575601.045568:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575601.045570:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575601.045572:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575601.045574:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045575:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575601.045577:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045579:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575601.045582:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575601.045586:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045588:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575601.045591:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575601.045594:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575601.045596:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575601.045599:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575601.045601:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575601.045610:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03b80.
00000020:00000040:2.0:1788575601.045613:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575601.045617:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575601.045619:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575601.045622:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575601.045624:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575601.045628:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575601.045636:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000100:00000001:2.0:1788575601.045642:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575601.045651:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575601.045661:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575601.045665:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575601.045668:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037d167ca] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575601.045684:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575601.045687:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575601.045689:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575601.045690:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575601.045691:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575601.045693:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575601.045695:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12532
00000400:00000200:2.0:1788575601.045698:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575601.045701:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575601.045703:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575601.045705:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045706:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575601.045708:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045710:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575601.045711:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575601.045712:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575601.045713:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575601.045715:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575601.045715:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575601.045719:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575601.045720:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1788575601.045721:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575601.045725:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575601.045725:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788575601.045728:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575601.045733:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575601.045734:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575601.045735:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575601.045739:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575601.045743:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045747:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575601.045750:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575601.045754:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1788575601.045755:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575601.045757:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575601.045758:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575601.045761:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575601.045762:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:2.0:1788575601.045764:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575601.045766:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:2.0:1788575601.045767:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575601.045771:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575601.045771:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575601.045778:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02d80.
00000100:00100000:3.0:1788575601.045779:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393221632:192.168.202.110@tcp:250:kworker.0
00000020:00000040:2.0:1788575601.045782:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575601.045788:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1788575601.045790:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575601.045791:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575601.045795:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575601.045795:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575601.045796:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575601.045799:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575601.045800:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575601.045804:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575601.045806:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
02000000:00000010:3.0:1788575601.045810:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
00000100:00000001:2.0:1788575601.045812:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575601.045813:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575601.045822:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575601.045822:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575601.045829:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:2.0:1788575601.045830:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575601.045834:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575601.045837:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009a9ccdb8] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788575601.045838:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575601.045847:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575601.045849:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393221632, portal 25
00000100:00080000:2.0:1788575601.045850:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575601.045855:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575601.045855:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575601.045859:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1788575601.045859:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00080000:2.0:1788575601.045860:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575601.045863:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575601.045866:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12532
00000100:00000040:3.0:1788575601.045871:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575601.045878:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575601.045883:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1788575601.045883:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575601.045889:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575601.045890:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1788575601.045893:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000040:2.0:1788575601.045894:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045898:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1788575601.045899:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393221632, offset 0
00000100:00000040:2.0:1788575601.045900:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045905:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575601.045907:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575601.045908:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575601.045911:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1788575601.045911:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:2.0:1788575601.045913:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575601.045915:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575601.045916:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575601.045919:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045923:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575601.045923:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575601.045927:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045928:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575601.045930:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575601.045931:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1788575601.045935:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575601.045937:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575601.045938:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:2.0:1788575601.045939:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575601.045943:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575601.045943:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:2.0:1788575601.045945:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575601.045948:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575601.045949:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00100000:2.0:1788575601.045950:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575601.045954:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575601.045956:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575601.045956:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000010:2.0:1788575601.045961:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03100.
00000020:00000040:2.0:1788575601.045965:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575601.045966:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330987:330987:256:4294967295] 192.168.202.110@tcp LPNI seq info [330944:330944:8:4294967295]
02000000:00000001:2.0:1788575601.045967:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575601.045969:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575601.045973:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575601.045975:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575601.045977:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575601.045980:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:2.0:1788575601.045985:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575601.045986:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1788575601.045993:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575601.046004:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1788575601.046011:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575601.046015:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575601.046016:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575601.046021:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008692924c] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1788575601.046023:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000100:00000001:2.0:1788575601.046033:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575601.046034:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575601.046037:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.0:1788575601.046040:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:2.0:1788575601.046042:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:3.1:1788575601.046049:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575601.046050:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12542)
00000800:00000200:3.1:1788575601.046055:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1788575601.046061:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575601.046064:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575601.046071:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575601.046076:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575601.046081:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575601.046084:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575601.046088:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575601.046092:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046102:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575601.046106:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575626 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046113:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575601.046115:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575601.046120:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575601.046125:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575601.046127:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307205561
00000100:00000010:3.0:1788575601.046133:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788575601.046139:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575601.046144:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307205561
00000100:00000010:3.0:1788575601.046147:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575601.046150:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575601.046153:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575601.046156:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575601.046158:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575601.046162:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575601/real 1788575601]  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575601.046172:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575601.046174:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575601.046176:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575601.046179:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575601.046181:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575601.046184:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575601.046187:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575601.046195:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046201:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575601.046203:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046206:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575601.046207:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575601.046211:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046218:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575601.046222:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046230:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575601.046232:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046234:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575601.046235:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046238:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575601.046240:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575601.046242:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575601.046246:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575601.046249:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575601.046253:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575601.046261:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393221632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575601.046268:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046270:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575601.046273:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393221632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046279:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575601.046282:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575601.046284:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575601.046287:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575601.046289:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575601.046290:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575601.046293:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575601.046295:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575601.046298:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575601.046300:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575601.046302:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:3.0:1788575601.046305:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575601.046306:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046309:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046315:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575601.046317:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575601.046319:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575601.046320:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575601.046322:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575601.046326:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046333:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575601.046334:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575601.046336:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575601.046338:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575601.046340:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575601.046343:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575601.046346:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575601.046348:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575601.046350:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575601.046352:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575601.046355:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575601.046357:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575601.046360:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575601.046363:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575601.046366:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575601.046370:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393221760, portal 10
00000100:00000001:3.0:1788575601.046372:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575601.046373:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:3.0:1788575601.046379:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046386:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575601.046388:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575601.046410:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575601.046413:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393221760, offset 0
00000400:00000200:3.0:1788575601.046417:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575601.046421:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575601.046425:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575601.046428:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575601.046431:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575601.046433:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575601.046435:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575601.046439:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330988:330988:256:4294967295] 192.168.202.110@tcp LPNI seq info [330945:330945:8:4294967295]
00000400:00000200:3.0:1788575601.046443:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575601.046449:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575601.046453:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575601.046456:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575601.046460:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575601.046462:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788575601.046466:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575601.046469:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1788575601.046473:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575601.046474:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575601.046477:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575601.046480:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575601.046483:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575601.046484:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575601.046501:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575601.046505:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046513:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575601.046516:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575626 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046522:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575601.046524:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575601.046527:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575601.046531:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575601.046534:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575601.046536:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307204647
00000100:00000001:3.0:1788575601.046539:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307205561
00000100:00000001:3.0:1788575601.046541:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307204647
00000100:00000001:3.0:1788575601.046543:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307205561
00000100:00000001:3.0:1788575601.046545:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575601.046547:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575601.046549:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575601.046553:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575601/real 1788575601]  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575601.046561:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575601.046562:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575601.046564:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575601.046567:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575601.046568:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575601.046570:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575601.046574:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575601.046582:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046588:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575601.046590:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046591:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575601.046593:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575601.046597:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046603:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575601.046607:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046614:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575601.046616:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046618:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575601.046619:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046621:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575601.046623:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575601.046625:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575601.046629:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575601.046631:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575601.046635:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575601.046643:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575601.046647:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575601.046650:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046663:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575601.046664:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575601.046667:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575601.046669:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575601.046671:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575601.046673:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221888:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575601.046677:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575601.046678:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575601.046680:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575601.046682:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575601.046686:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575601.046688:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575601.046691:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575601.046694:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575601.046697:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575601.046700:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393221888, portal 4
00000100:00000001:3.0:1788575601.046702:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575601.046703:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:3.0:1788575601.046709:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046716:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575601.046718:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575601.046720:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575601.046722:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393221888, offset 0
00000400:00000200:3.0:1788575601.046726:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575601.046729:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575601.046733:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575601.046736:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575601.046739:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575601.046741:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575601.046743:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575601.046746:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330989:330989:256:4294967295] 192.168.202.110@tcp LPNI seq info [330946:330946:8:4294967295]
00000400:00000200:3.0:1788575601.046749:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575601.046755:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575601.046759:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575601.046763:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575601.046766:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575601.046769:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788575601.046773:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575601.046775:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1788575601.046779:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575601.046781:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575601.046784:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575601.046787:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575601.046790:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575601.046792:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575601.046794:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575601.046797:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046804:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575601.046807:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575626 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046814:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575601.046816:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575601.046818:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575601.046822:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575601.046825:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575601.046827:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307204793
00000100:00000001:3.0:1788575601.046829:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307205561
00000100:00000001:3.0:1788575601.046832:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307204793
00000100:00000001:3.0:1788575601.046834:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307205561
00000100:00000001:3.0:1788575601.046836:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575601.046839:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575601.046840:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575601.046844:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575601/real 1788575601]  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575601.046852:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575601.046854:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575601.046856:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575601.046858:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575601.046860:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575601.046862:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575601.046865:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575601.046873:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046879:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575601.046881:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046883:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575601.046884:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575601.046887:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046893:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575601.046896:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046903:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575601.046905:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046906:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575601.046908:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046910:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575601.046911:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575601.046914:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575601.046917:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575601.046918:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575601.046922:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575601.046930:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393221888:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575601.046933:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046935:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575601.046939:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393221760/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046945:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575601.046947:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575601.046948:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575601.046951:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575601.046953:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575601.046954:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575601.046956:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575601.046958:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575601.046960:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575601.046961:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575601.046962:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:3.0:1788575601.046965:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575601.046966:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046968:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575601.046971:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393221888/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575626 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575601.046978:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575601.046979:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575601.046981:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575601.046983:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575601.046984:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575601.046986:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575601.046988:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575601.046989:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575601.046991:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575601.046993:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575601.046994:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:3.0:1788575601.046996:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575601.046998:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.046999:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575601.047003:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575601.047004:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575601.047006:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575601.047007:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575601.429510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575601.429516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575601.429520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575601.429521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575601.429523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575601.429524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575601.429618:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575601.429621:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575601.429630:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575601.557478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575601.557483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575601.557485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575601.557498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575601.557501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575601.557503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575601.557505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575601.557507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575601.557509:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575602.005513:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307205561, at 4307205568
00000100:00000040:0.0:1788575602.005521:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307205561, at 4307205568
00000100:00000010:0.0:1788575602.005532:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000010:1.0:1788575602.005537:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00010000:00000001:3.0:1788575602.453521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575602.453527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575602.453531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575602.453533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575602.453536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575602.453537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575602.453605:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575602.453609:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575602.453612:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575602.581483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575602.581489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575602.581503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575602.581507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575602.581511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575602.581515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575602.581517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575602.581520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575602.581522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575603.477480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575603.477487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575603.477492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575603.477493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575603.477508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575603.477509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575603.477559:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575603.477563:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575603.477566:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575603.605534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575603.605540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575603.605542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575603.605547:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575603.605550:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575603.605554:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575603.605557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575603.605559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575603.605562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575604.501546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575604.501552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575604.501557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575604.501559:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575604.501562:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575604.501563:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575604.501621:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575604.501629:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575604.501632:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575604.629536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575604.629543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575604.629545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575604.629550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575604.629553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575604.629557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575604.629560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575604.629562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575604.629565:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575605.525527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575605.525533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575605.525539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575605.525543:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575605.525545:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575605.525548:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575605.525550:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575605.525552:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575605.525553:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575605.525555:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575605.525557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575605.525558:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575605.525559:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575605.525560:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575605.525561:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575605.525562:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575605.525563:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575605.525564:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575605.525613:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575605.525616:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575605.525618:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575605.653494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575605.653500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575605.653524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575605.653530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575605.653533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575605.653536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575605.653538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575605.653540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575605.653542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575606.165493:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575606.165535:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575606.165538:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575606.165542:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575606.165547:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575606.165553:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12537
00000400:00000200:2.0:1788575606.165563:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575606.165573:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575606.165576:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575606.165579:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575606.165583:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575606.165585:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575606.165588:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575606.165591:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575606.165592:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575606.165595:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575606.165597:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575606.165598:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575606.165600:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575606.165602:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575606.165603:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575606.165604:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575606.165605:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575606.165607:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575606.165610:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575606.165612:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575606.165615:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575606.165618:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575606.165621:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575606.165623:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575606.165626:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575606.165635:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00e00.
00000020:00000040:2.0:1788575606.165647:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575606.165651:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575606.165653:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575606.165656:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575606.165659:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575606.165662:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575606.165670:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788575606.165677:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575606.165686:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575606.165695:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575606.165699:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575606.165703:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5f665f0] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575606.165721:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575606.165724:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575606.165728:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575606.165729:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575606.165731:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575606.165734:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575606.165738:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12537
00000400:00000200:2.0:1788575606.165742:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575606.165744:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:2.0:1788575606.165747:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1788575606.165749:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1788575606.165752:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575606.165752:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575606.165759:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575606.165760:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575606.165763:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575606.165769:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575606.165771:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575606.165773:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575606.165774:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575606.165778:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:2.0:1788575606.165779:0:186668:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:3.0:1788575606.165781:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1788575606.165782:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
02000000:00000001:3.0:1788575606.165784:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575606.165786:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:3.0:1788575606.165789:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393222016:192.168.202.110@tcp:250:kworker.0
00000020:00000040:2.0:1788575606.165790:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575606.165795:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575606.165796:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:2.0:1788575606.165800:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:3.0:1788575606.165800:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575606.165804:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575606.165806:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575606.165807:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788575606.165811:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000010:3.0:1788575606.165813:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
00000100:00000040:2.0:1788575606.165814:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575606.165816:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575606.165820:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575606.165824:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575606.165825:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000400:00000010:3.0:1788575606.165827:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00080000:2.0:1788575606.165831:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575606.165835:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575606.165835:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000001:2.0:1788575606.165841:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1788575606.165843:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393222016, portal 25
00000100:00000001:3.0:1788575606.165846:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1788575606.165847:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575606.165848:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
02000000:00000001:2.0:1788575606.165853:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575606.165855:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575606.165857:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575606.165863:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575606.165867:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00000001:2.0:1788575606.165868:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575606.165872:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575606.165873:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000010:2.0:1788575606.165877:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01c00.
00000400:00000010:3.0:1788575606.165878:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000020:00000040:2.0:1788575606.165882:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000200:3.0:1788575606.165884:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393222016, offset 0
02000000:00000001:2.0:1788575606.165886:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575606.165889:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575606.165891:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575606.165893:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575606.165895:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788575606.165896:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575606.165902:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000400:00000200:3.0:1788575606.165906:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575606.165907:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575606.165914:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:2.0:1788575606.165916:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575606.165920:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575606.165925:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575606.165930:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1788575606.165931:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575606.165934:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575606.165936:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575606.165939:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330990:330990:256:4294967295] 192.168.202.110@tcp LPNI seq info [330947:330947:8:4294967295]
00000100:00000040:2.0:1788575606.165944:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000de3acaa1] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575606.165945:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575606.165953:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575606.165958:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575606.165959:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1788575606.165964:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575606.165966:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575606.165969:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.0:1788575606.165969:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575606.165973:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000800:00000010:3.0:1788575606.165976:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000100:00080000:2.0:1788575606.165977:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12537
00000400:00000200:2.0:1788575606.165983:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:3.0:1788575606.165986:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:2.0:1788575606.165989:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:3.0:1788575606.165991:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00080000:2.0:1788575606.165994:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:3.1:1788575606.165998:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575606.166001:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575606.166003:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1788575606.166007:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000100:3.0:1788575606.166010:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575606.166013:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000100:00000040:2.0:1788575606.166014:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575606.166018:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575606.166018:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:2.0:1788575606.166024:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575606.166027:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:3.0:1788575606.166028:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1788575606.166032:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1788575606.166034:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000020:00000040:2.0:1788575606.166036:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000010:3.0:1788575606.166038:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000020:00000001:2.0:1788575606.166040:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:3.0:1788575606.166042:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000020:00000001:2.0:1788575606.166044:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:3.0:1788575606.166048:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575606.166051:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575606.166055:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575606.166059:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575606.166063:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575606.166068:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575606.166074:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575606.166075:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575606.166079:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575631 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575606.166080:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575606.166085:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575606.166088:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575606.166090:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575606.166091:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575606.166095:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:3.0:1788575606.166097:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788575606.166099:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575606.166104:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166104:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000010:2.0:1788575606.166110:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575606.166112:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575606.166114:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575606.166119:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575606.166120:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575606.166122:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575606.166123:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1788575606.166125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307210681
02000000:00000001:2.0:1788575606.166126:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575606.166130:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575606.166132:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
02000000:00000010:2.0:1788575606.166135:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000100:00000001:3.0:1788575606.166139:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575606.166140:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166146:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307210681
00000100:00100000:2.0:1788575606.166147:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:3.0:1788575606.166152:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:2.0:1788575606.166156:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575606.166158:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575606.166160:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166162:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575606.166165:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000389a8694] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166172:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575606.166175:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166177:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1788575606.166179:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1788575606.166185:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575606/real 1788575606]  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575606.166186:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575606.166193:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12547)
00000400:00000001:3.0:1788575606.166202:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575606.166204:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575606.166206:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575606.166210:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575606.166212:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575606.166214:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575606.166217:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575606.166223:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166227:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575606.166230:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166232:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575606.166234:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575606.166238:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166244:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575606.166248:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166255:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575606.166259:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166260:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575606.166262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166264:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575606.166266:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575606.166269:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575606.166273:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575606.166275:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575606.166277:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575606.166283:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393222016:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575606.166286:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166288:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575606.166291:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393222016/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166298:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575606.166300:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575606.166302:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575606.166304:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575606.166305:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575606.166306:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575606.166308:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575606.166310:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575606.166313:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575606.166315:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575606.166317:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:3.0:1788575606.166320:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575606.166321:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166323:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166330:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575606.166332:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166334:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575606.166335:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575606.166337:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575606.166341:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166348:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575606.166349:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575606.166351:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575606.166353:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575606.166355:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575606.166358:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222144:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575606.166361:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575606.166363:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575606.166365:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575606.166366:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575606.166369:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575606.166371:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575606.166375:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575606.166378:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575606.166381:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575606.166384:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393222144, portal 10
00000100:00000001:3.0:1788575606.166386:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575606.166387:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:3.0:1788575606.166412:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166419:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575606.166421:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575606.166424:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575606.166426:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393222144, offset 0
00000400:00000200:3.0:1788575606.166430:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575606.166434:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575606.166438:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575606.166440:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575606.166443:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575606.166445:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575606.166448:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575606.166450:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330991:330991:256:4294967295] 192.168.202.110@tcp LPNI seq info [330948:330948:8:4294967295]
00000400:00000200:3.0:1788575606.166454:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575606.166460:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575606.166465:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575606.166468:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575606.166472:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575606.166474:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1788575606.166478:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575606.166481:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1788575606.166484:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575606.166486:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575606.166489:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575606.166492:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575606.166495:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575606.166497:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575606.166499:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575606.166513:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166529:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575606.166532:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575631 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166544:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166545:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575606.166549:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575606.166552:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575606.166555:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575606.166557:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307209767
00000100:00000001:3.0:1788575606.166560:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307210681
00000100:00000001:3.0:1788575606.166562:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307209767
00000100:00000001:3.0:1788575606.166564:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307210681
00000100:00000001:3.0:1788575606.166565:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575606.166567:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166568:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575606.166571:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575606/real 1788575606]  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575606.166576:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575606.166576:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575606.166578:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575606.166579:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575606.166580:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575606.166582:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575606.166583:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575606.166588:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166592:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575606.166592:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166593:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575606.166594:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575606.166596:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166600:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575606.166603:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166607:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575606.166608:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166609:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575606.166609:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166610:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575606.166612:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575606.166614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575606.166617:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575606.166619:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575606.166622:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575606.166627:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222144:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575606.166630:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575606.166632:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166643:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575606.166643:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575606.166645:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575606.166646:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575606.166647:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575606.166649:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222272:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575606.166651:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575606.166652:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575606.166654:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575606.166655:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575606.166657:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575606.166658:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575606.166661:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575606.166662:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575606.166664:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575606.166666:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393222272, portal 4
00000100:00000001:3.0:1788575606.166667:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575606.166668:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:3.0:1788575606.166673:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166679:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575606.166682:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575606.166684:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575606.166686:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393222272, offset 0
00000400:00000200:3.0:1788575606.166689:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575606.166693:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575606.166696:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575606.166699:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575606.166701:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575606.166703:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575606.166705:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575606.166708:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330992:330992:256:4294967295] 192.168.202.110@tcp LPNI seq info [330949:330949:8:4294967295]
00000400:00000200:3.0:1788575606.166712:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575606.166718:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575606.166722:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575606.166725:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575606.166728:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575606.166730:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1788575606.166733:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575606.166734:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1788575606.166736:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575606.166738:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575606.166740:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575606.166742:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575606.166744:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575606.166745:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575606.166746:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575606.166749:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575631 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166754:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575606.166755:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575631 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166759:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166760:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575606.166762:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575606.166764:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575606.166766:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575606.166767:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307209913
00000100:00000001:3.0:1788575606.166769:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307210681
00000100:00000001:3.0:1788575606.166770:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307209913
00000100:00000001:3.0:1788575606.166772:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307210681
00000100:00000001:3.0:1788575606.166773:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575606.166774:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166775:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575606.166777:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575606/real 1788575606]  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575606.166781:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575606.166782:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575606.166783:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575606.166784:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575606.166785:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575606.166787:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575606.166789:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575606.166793:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166797:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575606.166797:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166798:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575606.166799:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575606.166801:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166805:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575606.166807:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166810:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575606.166811:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166812:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575606.166813:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166814:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575606.166815:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575606.166816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575606.166818:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575606.166820:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575606.166822:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575606.166826:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222272:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575606.166828:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166829:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575606.166831:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393222144/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166835:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575606.166836:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575606.166837:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575606.166838:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575606.166840:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575606.166840:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575606.166842:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575606.166843:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575606.166844:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575606.166845:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575606.166846:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:3.0:1788575606.166848:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575606.166849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166850:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575606.166852:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393222272/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575631 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575606.166855:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575606.166856:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575606.166857:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575606.166859:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575606.166860:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575606.166861:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575606.166863:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575606.166864:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575606.166866:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575606.166867:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575606.166868:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575606.166870:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575606.166871:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166873:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575606.166876:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575606.166878:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575606.166879:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575606.166881:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575606.549479:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575606.549486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575606.549491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575606.549492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575606.549495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575606.549496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575606.549549:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575606.549552:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575606.549554:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575606.677515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575606.677522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575606.677524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575606.677528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575606.677531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575606.677533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575606.677535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575606.677537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575606.677539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575607.125498:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307210681, at 4307210688
00000100:00000040:1.0:1788575607.125498:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307210681, at 4307210688
00000100:00000010:1.0:1788575607.125522:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:0.0:1788575607.125524:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575607.573532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575607.573541:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575607.573543:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575607.573546:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575607.573551:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575607.573553:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575607.573556:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575607.573558:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575607.573562:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575607.573564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575607.573567:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575607.573569:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575607.573659:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575607.573662:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575607.573665:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575607.701537:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575607.701544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575607.701546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575607.701552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575607.701556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575607.701559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575607.701562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575607.701565:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575607.701568:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575608.597564:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575608.597571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575608.597577:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575608.597578:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575608.597581:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575608.597583:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575608.597649:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575608.597652:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575608.597656:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575608.725508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575608.725527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575608.725530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575608.725536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575608.725540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575608.725543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575608.725545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575608.725548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575608.725551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575609.621499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575609.621506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575609.621511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575609.621542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575609.621547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575609.621548:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575609.621593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575609.621595:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575609.621598:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575609.749506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575609.749533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575609.749535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575609.749540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575609.749544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575609.749548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575609.749551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575609.749553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575609.749556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575610.645530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575610.645539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575610.645545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575610.645546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575610.645549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575610.645551:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575610.645610:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575610.645614:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575610.645617:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575610.773602:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575610.773609:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575610.773611:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575610.773616:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575610.773619:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575610.773623:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575610.773625:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575610.773627:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575610.773630:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575611.285493:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575611.285503:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575611.285505:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575611.285509:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575611.285513:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575611.285532:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12542
00000400:00000200:0.0:1788575611.285541:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575611.285547:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575611.285550:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575611.285553:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575611.285555:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575611.285558:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575611.285560:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575611.285564:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575611.285565:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575611.285568:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575611.285571:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575611.285573:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575611.285575:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575611.285579:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575611.285581:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575611.285583:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575611.285584:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575611.285587:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575611.285590:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575611.285592:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575611.285595:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575611.285598:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575611.285601:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575611.285603:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575611.285605:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575611.285613:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788575611.285617:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575611.285621:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575611.285623:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575611.285626:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575611.285628:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575611.285633:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575611.285639:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575611.285646:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575611.285653:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575611.285662:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575611.285666:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575611.285670:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575611.285692:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575611.285695:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575611.285698:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575611.285699:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575611.285701:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575611.285704:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575611.285707:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12542
00000400:00000200:0.0:1788575611.285711:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575611.285714:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575611.285716:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575611.285720:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575611.285723:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575611.285723:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575611.285728:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1788575611.285729:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575611.285732:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788575611.285733:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575611.285738:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:3.0:1788575611.285739:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575611.285740:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575611.285745:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575611.285749:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575611.285750:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1788575611.285752:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575611.285753:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575611.285758:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:3.0:1788575611.285759:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575611.285762:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1788575611.285766:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575611.285766:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575611.285772:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1788575611.285772:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393222400:192.168.202.110@tcp:250:kworker.0
00000100:00000040:0.0:1788575611.285778:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575611.285779:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575611.285783:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1788575611.285784:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575611.285785:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575611.285789:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:0.0:1788575611.285790:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000010:3.0:1788575611.285794:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
00000100:00080000:0.0:1788575611.285796:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575611.285797:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575611.285802:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575611.285804:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575611.285805:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575611.285809:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
02000000:00000001:0.0:1788575611.285811:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575611.285814:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575611.285814:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:0.0:1788575611.285818:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:3.0:1788575611.285821:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393222400, portal 25
00000008:00100000:0.0:1788575611.285822:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575611.285824:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575611.285827:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523500544 : -130484186051072 : ffff895348710e00)
00000008:00000001:0.0:1788575611.285828:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575611.285831:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575611.285833:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1788575611.285837:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000100:00000001:3.0:1788575611.285839:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:0.0:1788575611.285842:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000040:3.0:1788575611.285843:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575611.285847:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
02000000:00000001:0.0:1788575611.285848:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575611.285851:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575611.285852:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393222400, offset 0
00000100:00000001:0.0:1788575611.285853:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575611.285856:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575611.285858:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575611.285859:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000010:0.0:1788575611.285863:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575611.285868:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575611.285868:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575611.285874:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:0.0:1788575611.285877:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575611.285880:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575611.285885:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575611.285891:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788575611.285895:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575611.285895:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788575611.285901:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575611.285901:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330993:330993:256:4294967295] 192.168.202.110@tcp LPNI seq info [330950:330950:8:4294967295]
00000400:00000200:3.0:1788575611.285908:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575611.285910:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575611.285920:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575611.285928:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575611.285931:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575611.285933:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000100:00080000:0.0:1788575611.285936:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.0:1788575611.285941:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1788575611.285943:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:3.0:1788575611.285945:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1788575611.285946:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575611.285952:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.1:1788575611.285952:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575611.285957:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1788575611.285958:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000100:3.0:1788575611.285962:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575611.285965:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000100:00080000:0.0:1788575611.285966:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12542
00000400:00000200:3.0:1788575611.285970:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575611.285975:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575611.285978:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575611.285983:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000200:0.0:1788575611.285984:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:3.0:1788575611.285986:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00080000:0.0:1788575611.285988:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575611.285991:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:0.0:1788575611.285996:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:3.0:1788575611.285996:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575611.286001:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575611.286007:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575611.286008:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575611.286011:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575611.286013:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575611.286018:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1788575611.286022:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575611.286023:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:3.0:1788575611.286025:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000001:0.0:1788575611.286027:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575611.286030:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:3.0:1788575611.286031:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575611.286032:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575611.286037:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1788575611.286038:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:3.0:1788575611.286040:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307215801
00000100:00000040:0.0:1788575611.286044:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000010:3.0:1788575611.286046:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:0.0:1788575611.286047:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575611.286052:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575611.286054:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575611.286055:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000040:0.0:1788575611.286056:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575611.286062:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575611.286064:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286064:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307215801
02000000:00000001:0.0:1788575611.286069:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:3.0:1788575611.286071:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
02000000:00000001:0.0:1788575611.286073:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575611.286077:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286077:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788575611.286082:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575611.286084:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00100000:0.0:1788575611.286085:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575611.286091:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286091:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575611.286096:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286097:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000010:0.0:1788575611.286101:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788575611.286105:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00100000:3.0:1788575611.286106:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575611/real 1788575611]  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575611.286110:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575611.286112:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575611.286116:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575611.286118:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575611.286121:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1788575611.286124:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000010:0.0:1788575611.286127:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000400:00000001:3.0:1788575611.286128:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575611.286132:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000100:00000001:0.0:1788575611.286134:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575611.286136:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575611.286141:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00100000:0.0:1788575611.286142:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575611.286145:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575611.286152:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575611.286153:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575611.286158:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575611.286166:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575611.286171:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575611.286179:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575611.286184:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575611.286186:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:0.0:1788575611.286190:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575611.286191:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286196:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:0.0:1788575611.286197:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12552)
00000100:00000001:3.0:1788575611.286199:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575611.286205:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286210:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575611.286215:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286219:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575611.286221:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286222:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575611.286223:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286224:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575611.286225:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575611.286227:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575611.286230:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575611.286231:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575611.286233:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575611.286238:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393222400:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575611.286241:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286242:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575611.286244:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710e00 x1875444393222400/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286247:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575611.286249:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575611.286250:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575611.286253:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575611.286254:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575611.286255:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575611.286256:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575611.286258:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:3.0:1788575611.286260:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575611.286262:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575611.286263:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710e00.
00000100:00000001:3.0:1788575611.286266:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575611.286268:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286270:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286277:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575611.286279:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286280:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575611.286282:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575611.286283:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575611.286287:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286294:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575611.286296:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575611.286298:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575611.286300:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575611.286302:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575611.286305:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575611.286308:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575611.286310:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575611.286311:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575611.286312:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575611.286314:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575611.286315:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575611.286318:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575611.286320:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575611.286322:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575611.286324:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393222528, portal 10
00000100:00000001:3.0:1788575611.286326:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575611.286326:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523505920 : -130484186045696 : ffff895348712300)
00000100:00000040:3.0:1788575611.286331:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286335:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575611.286336:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575611.286337:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575611.286339:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393222528, offset 0
00000400:00000200:3.0:1788575611.286341:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575611.286344:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575611.286346:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575611.286348:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575611.286349:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575611.286351:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575611.286352:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575611.286354:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330994:330994:256:4294967295] 192.168.202.110@tcp LPNI seq info [330951:330951:8:4294967295]
00000400:00000200:3.0:1788575611.286357:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575611.286361:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575611.286364:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575611.286366:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575611.286368:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575611.286370:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575611.286373:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575611.286375:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575611.286377:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575611.286378:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575611.286380:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575611.286382:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575611.286384:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575611.286385:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575611.286386:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575611.286388:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286409:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575611.286411:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286415:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286416:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575611.286418:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575611.286420:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575611.286422:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575611.286424:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307214887
00000100:00000001:3.0:1788575611.286425:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307215801
00000100:00000001:3.0:1788575611.286427:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307214887
00000100:00000001:3.0:1788575611.286428:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307215801
00000100:00000001:3.0:1788575611.286429:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575611.286431:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286432:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575611.286434:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575611/real 1788575611]  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575611.286439:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575611.286440:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575611.286441:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575611.286443:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575611.286443:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575611.286444:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575611.286446:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575611.286451:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286455:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575611.286455:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286456:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575611.286457:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575611.286459:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286466:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575611.286469:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286472:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575611.286473:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286474:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575611.286475:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286476:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575611.286478:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575611.286479:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575611.286481:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575611.286483:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575611.286485:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575611.286490:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575611.286492:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575611.286494:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286497:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575611.286498:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575611.286499:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575611.286500:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575611.286501:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575611.286503:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575611.286504:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575611.286505:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575611.286506:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575611.286507:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575611.286509:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575611.286510:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575611.286512:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575611.286514:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575611.286516:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575611.286529:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393222656, portal 4
00000100:00000001:3.0:1788575611.286532:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575611.286533:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:3.0:1788575611.286539:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286544:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575611.286547:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575611.286549:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575611.286552:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393222656, offset 0
00000400:00000200:3.0:1788575611.286556:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575611.286560:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575611.286563:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575611.286565:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575611.286568:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575611.286570:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575611.286573:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575611.286575:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330995:330995:256:4294967295] 192.168.202.110@tcp LPNI seq info [330952:330952:8:4294967295]
00000400:00000200:3.0:1788575611.286577:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575611.286581:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575611.286585:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575611.286587:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575611.286589:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575611.286591:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575611.286593:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575611.286595:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575611.286597:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575611.286599:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575611.286601:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575611.286603:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575611.286604:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575611.286605:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575611.286607:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575611.286609:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575636 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286614:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575611.286616:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575636 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286620:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286621:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575611.286623:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575611.286624:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575611.286626:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575611.286628:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307215033
00000100:00000001:3.0:1788575611.286629:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307215801
00000100:00000001:3.0:1788575611.286631:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307215033
00000100:00000001:3.0:1788575611.286632:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307215801
00000100:00000001:3.0:1788575611.286633:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575611.286634:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286635:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575611.286637:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575611/real 1788575611]  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575611.286641:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575611.286642:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575611.286643:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575611.286644:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575611.286645:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575611.286646:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575611.286648:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575611.286653:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286656:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575611.286657:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286658:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575611.286659:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575611.286661:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286664:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575611.286666:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286669:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575611.286671:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286672:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575611.286673:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286674:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575611.286675:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575611.286677:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575611.286678:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575611.286679:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575611.286681:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575611.286685:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575611.286687:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286689:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575611.286690:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712300 x1875444393222528/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286694:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575611.286695:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575611.286696:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575611.286697:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575611.286698:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575611.286699:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575611.286700:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575611.286701:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:3.0:1788575611.286703:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575611.286704:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575611.286705:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712300.
00000100:00000001:3.0:1788575611.286707:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575611.286708:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286708:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575611.286710:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393222656/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575636 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575611.286718:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575611.286718:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575611.286719:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575611.286720:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575611.286721:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575611.286722:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575611.286723:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575611.286723:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:3.0:1788575611.286724:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575611.286725:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575611.286725:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:3.0:1788575611.286727:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575611.286727:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286728:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575611.286730:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575611.286731:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575611.286732:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575611.286733:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575611.669512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575611.669527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575611.669532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575611.669533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575611.669535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575611.669536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575611.669595:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575611.669598:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575611.669601:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575611.797461:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575611.797466:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575611.797468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575611.797479:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575611.797482:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575611.797484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575611.797486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575611.797488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575611.797490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575612.245486:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307215801, at 4307215808
00000100:00000040:0.0:1788575612.245489:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307215801, at 4307215808
00000100:00000010:1.0:1788575612.245499:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575612.245501:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00010000:00000001:3.0:1788575612.693548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575612.693554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575612.693559:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575612.693560:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575612.693564:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575612.693565:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575612.693630:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575612.693633:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575612.693637:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575612.821501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575612.821506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575612.821507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575612.821511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575612.821514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575612.821517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575612.821519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575612.821520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575612.821535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575613.717499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575613.717505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575613.717510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575613.717511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575613.717514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575613.717515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575613.717583:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575613.717587:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575613.717591:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575613.845555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575613.845562:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575613.845564:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575613.845569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575613.845572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575613.845576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575613.845579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575613.845582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575613.845585:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575614.742215:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575614.742221:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575614.742226:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575614.742229:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575614.742231:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575614.742235:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575614.742237:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575614.742239:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575614.742241:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575614.742243:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575614.742246:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575614.742247:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575614.742249:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575614.742250:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575614.742251:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575614.742252:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575614.742254:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575614.742256:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575614.742298:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575614.742302:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575614.742305:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575614.869486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575614.869492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575614.869493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575614.869497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575614.869500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575614.869503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575614.869505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575614.869506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575614.869508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575615.765485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575615.765491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575615.765497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575615.765498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575615.765501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575615.765503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575615.765556:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575615.765560:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575615.765563:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575615.893509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575615.893517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575615.893520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575615.893526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575615.893530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575615.893545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575615.893548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575615.893551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575615.893555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575616.405499:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575616.405509:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575616.405512:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575616.405516:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575616.405521:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575616.405527:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12547
00000400:00000200:0.0:1788575616.405546:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575616.405553:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575616.405556:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575616.405560:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405562:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575616.405564:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405566:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575616.405569:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575616.405570:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575616.405579:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575616.405582:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575616.405583:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575616.405584:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575616.405586:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405588:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575616.405590:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405591:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575616.405593:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575616.405596:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405597:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575616.405599:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575616.405602:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575616.405603:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575616.405605:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575616.405607:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575616.405614:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575616.405618:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575616.405621:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575616.405623:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575616.405629:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575616.405631:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575616.405635:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575616.405642:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788575616.405648:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575616.405657:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575616.405667:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575616.405671:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575616.405676:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575616.405697:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575616.405700:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575616.405702:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575616.405703:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575616.405704:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575616.405706:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575616.405708:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12547
00000400:00000200:0.0:1788575616.405711:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575616.405713:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575616.405715:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575616.405717:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405719:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575616.405720:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405722:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575616.405724:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575616.405725:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575616.405726:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575616.405728:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575616.405728:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575616.405732:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788575616.405733:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575616.405735:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1788575616.405737:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575616.405743:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575616.405744:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405749:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575616.405753:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405758:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575616.405762:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1788575616.405763:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575616.405765:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575616.405769:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575616.405771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575616.405773:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575616.405775:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575616.405777:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575616.405781:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575616.405785:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1788575616.405787:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393222784:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788575616.405788:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575616.405792:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575616.405797:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575616.405797:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:0.0:1788575616.405801:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575616.405802:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575616.405804:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:0.0:1788575616.405805:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
02000000:00000001:3.0:1788575616.405808:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575616.405809:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575616.405814:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575616.405814:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1788575616.405818:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575616.405820:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:3.0:1788575616.405821:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788575616.405822:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575616.405826:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575616.405827:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575616.405834:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1788575616.405835:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000400:00000010:3.0:1788575616.405840:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:0.0:1788575616.405842:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575616.405850:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575616.405858:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575616.405860:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575616.405864:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575616.405871:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575616.405873:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1788575616.405874:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00080000:0.0:1788575616.405876:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575616.405879:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575616.405880:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575616.405882:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000200:3.0:1788575616.405883:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393222784, portal 25
00000100:00080000:0.0:1788575616.405885:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12547
00000100:00000001:3.0:1788575616.405888:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575616.405890:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000400:00000200:0.0:1788575616.405891:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575616.405895:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575616.405897:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575616.405901:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575616.405901:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575616.405905:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575616.405909:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405913:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575616.405916:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575616.405918:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575616.405921:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575616.405922:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:3.0:1788575616.405922:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:0.0:1788575616.405925:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575616.405929:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575616.405930:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788575616.405932:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405933:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575616.405934:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405938:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575616.405941:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:3.0:1788575616.405941:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00080000:0.0:1788575616.405943:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405946:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575616.405948:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1788575616.405949:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393222784, offset 0
02000000:00000001:0.0:1788575616.405950:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575616.405954:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575616.405955:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575616.405957:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575616.405959:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575616.405960:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575616.405961:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575616.405966:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788575616.405968:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575616.405972:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1788575616.405972:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575616.405973:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575616.405976:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575616.405977:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575616.405979:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575616.405981:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000010:0.0:1788575616.405984:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000400:00000200:3.0:1788575616.405987:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575616.405988:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575616.405993:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00100000:0.0:1788575616.405994:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575616.405999:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575616.406004:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788575616.406005:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575616.406008:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575616.406012:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330996:330996:256:4294967295] 192.168.202.110@tcp LPNI seq info [330953:330953:8:4294967295]
00000100:00000040:0.0:1788575616.406014:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575616.406022:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575616.406029:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575616.406034:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575616.406038:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1788575616.406041:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:3.0:1788575616.406046:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788575616.406048:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12557)
00000800:00000010:3.0:1788575616.406052:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575616.406059:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575616.406064:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788575616.406069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575616.406073:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1788575616.406081:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575616.406083:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575616.406089:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575616.406093:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575616.406098:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575616.406100:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575616.406104:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575616.406108:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406117:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575616.406121:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406125:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575616.406126:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575616.406131:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575616.406133:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575616.406136:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307220921
00000100:00000010:3.0:1788575616.406141:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575616.406147:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575616.406158:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307220921
00000100:00000010:3.0:1788575616.406162:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575616.406165:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575616.406169:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575616.406173:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575616.406175:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575616.406181:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575616/real 1788575616]  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575616.406188:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575616.406189:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575616.406190:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575616.406192:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575616.406193:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575616.406195:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575616.406197:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575616.406202:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406206:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575616.406208:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406210:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575616.406212:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575616.406216:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406222:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575616.406227:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406234:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575616.406237:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406238:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575616.406240:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406242:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575616.406244:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575616.406247:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575616.406250:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575616.406252:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575616.406254:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575616.406259:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393222784:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575616.406263:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406264:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575616.406266:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393222784/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406274:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575616.406276:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575616.406277:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575616.406279:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575616.406281:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575616.406282:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575616.406283:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575616.406285:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5c00.
02000000:00000001:3.0:1788575616.406287:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575616.406289:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575616.406290:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:3.0:1788575616.406292:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575616.406292:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406294:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406299:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575616.406300:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575616.406301:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575616.406302:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575616.406303:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575616.406306:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406310:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575616.406311:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575616.406312:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575616.406313:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575616.406314:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575616.406316:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222912:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575616.406318:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575616.406319:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575616.406320:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575616.406321:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575616.406323:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575616.406324:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575616.406327:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575616.406329:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575616.406331:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575616.406333:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393222912, portal 10
00000100:00000001:3.0:1788575616.406334:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575616.406335:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00000040:3.0:1788575616.406339:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406343:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575616.406344:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575616.406346:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575616.406347:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393222912, offset 0
00000400:00000200:3.0:1788575616.406349:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575616.406351:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575616.406354:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575616.406356:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575616.406357:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575616.406358:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575616.406360:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575616.406363:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330997:330997:256:4294967295] 192.168.202.110@tcp LPNI seq info [330954:330954:8:4294967295]
00000400:00000200:3.0:1788575616.406366:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575616.406372:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575616.406376:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575616.406379:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575616.406382:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575616.406385:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788575616.406388:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575616.406408:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1788575616.406413:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575616.406415:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575616.406418:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575616.406421:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575616.406424:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575616.406426:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575616.406429:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575616.406432:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406440:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575616.406448:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406454:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575616.406456:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575616.406459:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575616.406462:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575616.406465:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575616.406467:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307220007
00000100:00000001:3.0:1788575616.406470:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307220921
00000100:00000001:3.0:1788575616.406472:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307220007
00000100:00000001:3.0:1788575616.406474:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307220921
00000100:00000001:3.0:1788575616.406476:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575616.406478:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575616.406480:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575616.406483:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575616/real 1788575616]  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575616.406491:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575616.406492:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575616.406494:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575616.406496:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575616.406497:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575616.406499:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575616.406502:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575616.406507:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406514:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575616.406515:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406516:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575616.406517:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575616.406519:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406522:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575616.406524:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406528:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575616.406530:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406531:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575616.406543:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406546:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575616.406548:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575616.406550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575616.406553:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575616.406556:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575616.406560:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575616.406568:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393222912:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575616.406571:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575616.406573:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406577:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575616.406578:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575616.406580:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575616.406581:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575616.406582:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575616.406584:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223040:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575616.406587:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575616.406589:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575616.406590:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575616.406592:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575616.406595:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575616.406597:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575616.406601:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575616.406604:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575616.406607:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575616.406610:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393223040, portal 4
00000100:00000001:3.0:1788575616.406612:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575616.406613:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:3.0:1788575616.406618:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406625:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575616.406627:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575616.406629:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575616.406632:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393223040, offset 0
00000400:00000200:3.0:1788575616.406636:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575616.406640:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575616.406651:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575616.406654:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575616.406656:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575616.406658:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575616.406661:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575616.406664:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330998:330998:256:4294967295] 192.168.202.110@tcp LPNI seq info [330955:330955:8:4294967295]
00000400:00000200:3.0:1788575616.406668:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575616.406674:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575616.406679:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575616.406682:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575616.406686:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575616.406688:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1788575616.406692:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575616.406695:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1788575616.406699:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575616.406700:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575616.406703:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575616.406706:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575616.406709:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575616.406711:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575616.406713:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575616.406716:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406724:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575616.406727:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406733:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575616.406735:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575616.406737:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575616.406740:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575616.406743:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575616.406745:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307220153
00000100:00000001:3.0:1788575616.406748:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307220921
00000100:00000001:3.0:1788575616.406750:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307220153
00000100:00000001:3.0:1788575616.406752:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307220921
00000100:00000001:3.0:1788575616.406754:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575616.406756:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575616.406758:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575616.406761:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575616/real 1788575616]  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575616.406768:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575616.406769:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575616.406771:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575616.406773:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575616.406774:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575616.406776:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575616.406779:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575616.406788:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406794:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575616.406796:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406797:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575616.406799:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575616.406802:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406809:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575616.406813:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406820:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575616.406822:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406824:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575616.406825:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406827:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575616.406828:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575616.406831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575616.406833:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575616.406836:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575616.406839:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575616.406847:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223040:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575616.406851:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406852:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575616.406855:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393222912/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406862:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575616.406863:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575616.406865:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575616.406867:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575616.406869:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575616.406870:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575616.406872:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575616.406874:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:3.0:1788575616.406876:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575616.406878:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575616.406879:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:3.0:1788575616.406881:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575616.406882:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406884:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575616.406887:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393223040/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575616.406893:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575616.406894:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575616.406896:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575616.406898:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575616.406899:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575616.406900:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575616.406901:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575616.406903:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:3.0:1788575616.406905:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575616.406906:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575616.406907:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:3.0:1788575616.406909:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575616.406910:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406912:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575616.406915:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575616.406916:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575616.406918:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575616.406919:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575616.789522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575616.789529:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575616.789532:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575616.789564:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575616.789569:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575616.789571:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575616.789639:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575616.789642:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575616.789646:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575616.789648:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575616.789651:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575616.789652:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575616.789712:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575616.789716:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575616.789719:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575616.917510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575616.917518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575616.917521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575616.917526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575616.917530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575616.917533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575616.917563:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575616.917566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575616.917569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575617.365579:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307220921, at 4307220928
00000100:00000040:1.0:1788575617.365579:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307220921, at 4307220928
00000100:00000010:0.0:1788575617.365605:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:1.0:1788575617.365607:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575617.813516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575617.813568:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575617.813573:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575617.813574:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575617.813577:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575617.813578:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575617.813638:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575617.813641:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575617.813643:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575617.941558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575617.941564:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575617.941565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575617.941569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575617.941572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575617.941575:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575617.941577:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575617.941579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575617.941581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575618.837513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575618.837527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575618.837536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575618.837537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575618.837560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575618.837561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575618.837629:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575618.837681:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575618.837684:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575618.965603:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575618.965618:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575618.965620:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575618.965626:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575618.965629:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575618.965633:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575618.965635:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575618.965638:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575618.965640:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575619.861523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575619.861530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575619.861536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575619.861537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575619.861540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575619.861541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575619.861630:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575619.861634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575619.861637:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575619.989523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575619.989529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575619.989531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575619.989536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575619.989539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575619.989551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575619.989555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575619.989557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575619.989560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575620.885468:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575620.885474:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575620.885477:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575620.885478:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575620.885480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575620.885481:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575620.885528:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575620.885532:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575620.885536:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575621.013634:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575621.013641:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575621.013644:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575621.013649:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575621.013652:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575621.013656:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575621.013670:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575621.013673:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575621.013689:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575621.525542:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575621.525575:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575621.525578:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575621.525582:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575621.525586:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575621.525591:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12552
00000400:00000200:0.0:1788575621.525598:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575621.525603:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575621.525605:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575621.525607:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525610:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575621.525611:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525613:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575621.525615:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575621.525617:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575621.525618:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575621.525621:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575621.525622:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575621.525623:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575621.525625:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525626:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575621.525627:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525628:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575621.525630:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575621.525633:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525635:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575621.525637:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575621.525639:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575621.525641:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575621.525642:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575621.525644:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575621.525650:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788575621.525653:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575621.525655:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575621.525656:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575621.525659:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575621.525661:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575621.525664:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575621.525669:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788575621.525674:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575621.525680:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575621.525686:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575621.525689:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575621.525693:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575621.525709:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575621.525713:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575621.525716:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575621.525718:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575621.525719:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575621.525722:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575621.525737:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12552
00000400:00000200:0.0:1788575621.525741:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575621.525744:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575621.525746:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575621.525747:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1788575621.525749:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525750:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575621.525750:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1788575621.525752:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575621.525752:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788575621.525754:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575621.525756:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575621.525757:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000040:3.0:1788575621.525758:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1788575621.525759:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575621.525760:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575621.525762:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575621.525765:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575621.525766:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1788575621.525767:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575621.525768:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575621.525769:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575621.525770:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575621.525771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575621.525772:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575621.525774:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575621.525774:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575621.525776:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575621.525777:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525779:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575621.525779:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393223168:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788575621.525781:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575621.525783:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575621.525785:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575621.525787:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575621.525787:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1788575621.525789:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575621.525789:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575621.525791:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1788575621.525793:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575621.525795:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575621.525796:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
00000100:00000001:0.0:1788575621.525797:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575621.525798:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575621.525800:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575621.525801:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:3.0:1788575621.525803:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575621.525804:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575621.525805:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575621.525806:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:0.0:1788575621.525807:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575621.525808:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575621.525809:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575621.525813:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000400:00000010:3.0:1788575621.525813:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:0.0:1788575621.525817:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575621.525818:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393223168, portal 25
00000100:00100000:0.0:1788575621.525821:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575621.525821:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575621.525823:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
00000100:00000001:0.0:1788575621.525827:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1788575621.525828:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575621.525829:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575621.525833:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.525835:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575621.525838:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525841:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575621.525841:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00080000:0.0:1788575621.525844:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1788575621.525844:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393223168, offset 0
00000100:00080000:0.0:1788575621.525846:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575621.525848:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575621.525849:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575621.525850:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525851:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575621.525853:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12552
00000400:00000200:0.0:1788575621.525856:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575621.525858:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575621.525859:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575621.525861:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575621.525863:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575621.525863:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788575621.525865:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575621.525867:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525868:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575621.525868:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:0.0:1788575621.525871:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575621.525872:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:3.0:1788575621.525872:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:0.0:1788575621.525874:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575621.525875:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575621.525876:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:0.0:1788575621.525877:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575621.525878:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:0.0:1788575621.525880:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575621.525882:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575621.525882:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [330999:330999:256:4294967295] 192.168.202.110@tcp LPNI seq info [330956:330956:8:4294967295]
00000100:00000001:0.0:1788575621.525884:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575621.525886:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575621.525887:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525889:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575621.525891:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575621.525893:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525895:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575621.525896:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1788575621.525897:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575621.525899:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575621.525901:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575621.525902:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575621.525903:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575621.525905:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000010:3.0:1788575621.525906:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000008:00000001:0.0:1788575621.525907:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575621.525908:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575621.525911:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788575621.525913:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:3.0:1788575621.525913:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1788575621.525915:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.0:1788575621.525916:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:0.0:1788575621.525917:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575621.525918:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575621.525920:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575621.525921:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575621.525922:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:0.0:1788575621.525924:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000800:00000200:3.1:1788575621.525925:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1788575621.525927:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1788575621.525929:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00100000:0.0:1788575621.525931:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:3.0:1788575621.525931:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000100:00000001:0.0:1788575621.525936:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575621.525937:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1788575621.525938:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575621.525941:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575621.525941:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575621.525946:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575621.525949:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575621.525953:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:0.0:1788575621.525957:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575621.525957:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575621.525959:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575621.525963:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575621.525965:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575621.525968:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12562)
00000100:00000040:3.0:1788575621.525968:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575646 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.525975:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575621.525977:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575621.525983:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575621.525987:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575621.525990:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307226041
00000100:00000010:3.0:1788575621.526008:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575621.526013:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575621.526022:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307226041
00000100:00000010:3.0:1788575621.526026:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575621.526029:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575621.526032:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575621.526035:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575621.526037:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575621.526042:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575621/real 1788575621]  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575621.526051:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575621.526053:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575621.526055:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575621.526058:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575621.526060:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575621.526062:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575621.526065:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575621.526074:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526081:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575621.526082:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526085:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575621.526087:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575621.526091:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526098:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575621.526101:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526106:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575621.526108:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526109:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575621.526109:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526111:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575621.526112:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575621.526114:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575621.526116:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575621.526117:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575621.526120:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575621.526124:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393223168:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575621.526127:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526129:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575621.526131:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393223168/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526135:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575621.526136:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575621.526138:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575621.526140:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575621.526141:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575621.526142:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575621.526144:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575621.526146:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:3.0:1788575621.526148:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575621.526149:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575621.526150:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:3.0:1788575621.526152:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575621.526153:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526155:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526161:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575621.526162:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575621.526163:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575621.526164:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575621.526165:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575621.526167:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526171:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575621.526172:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575621.526173:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575621.526174:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575621.526175:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575621.526176:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223296:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575621.526178:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575621.526180:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575621.526181:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575621.526182:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575621.526183:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575621.526185:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575621.526187:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575621.526189:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575621.526191:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575621.526193:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393223296, portal 10
00000100:00000001:3.0:1788575621.526202:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575621.526203:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000040:3.0:1788575621.526207:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526211:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575621.526212:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575621.526214:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575621.526215:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393223296, offset 0
00000400:00000200:3.0:1788575621.526217:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575621.526220:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575621.526222:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575621.526224:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575621.526226:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575621.526227:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575621.526229:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575621.526231:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331000:331000:256:4294967295] 192.168.202.110@tcp LPNI seq info [330957:330957:8:4294967295]
00000400:00000200:3.0:1788575621.526233:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575621.526237:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575621.526240:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575621.526242:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575621.526244:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575621.526246:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575621.526249:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575621.526250:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575621.526253:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575621.526254:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575621.526255:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575621.526258:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575621.526260:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575621.526261:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575621.526262:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575621.526264:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526269:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575621.526270:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575646 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526274:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575621.526275:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575621.526277:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575621.526279:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575621.526281:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575621.526282:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307225126
00000100:00000001:3.0:1788575621.526284:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307226041
00000100:00000001:3.0:1788575621.526285:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307225126
00000100:00000001:3.0:1788575621.526287:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307226041
00000100:00000001:3.0:1788575621.526288:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575621.526290:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575621.526291:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575621.526295:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575621/real 1788575621]  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575621.526301:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575621.526303:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575621.526304:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575621.526306:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575621.526307:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575621.526309:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575621.526312:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575621.526319:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526325:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575621.526327:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526328:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575621.526330:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575621.526333:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526340:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575621.526344:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526352:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575621.526353:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526355:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575621.526356:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526357:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575621.526358:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575621.526360:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575621.526362:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575621.526363:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575621.526365:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575621.526370:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223296:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575621.526372:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575621.526374:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526377:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575621.526378:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575621.526380:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575621.526381:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575621.526382:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575621.526384:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575621.526385:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575621.526387:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575621.526388:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575621.526389:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575621.526405:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575621.526407:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575621.526409:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575621.526411:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575621.526413:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575621.526415:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393223424, portal 4
00000100:00000001:3.0:1788575621.526417:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575621.526418:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:3.0:1788575621.526421:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526425:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575621.526426:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575621.526428:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575621.526429:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393223424, offset 0
00000400:00000200:3.0:1788575621.526431:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575621.526433:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575621.526435:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575621.526437:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575621.526438:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575621.526440:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575621.526441:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575621.526443:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331001:331001:256:4294967295] 192.168.202.110@tcp LPNI seq info [330958:330958:8:4294967295]
00000400:00000200:3.0:1788575621.526445:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575621.526449:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575621.526452:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575621.526453:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575621.526456:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575621.526457:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575621.526460:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575621.526461:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575621.526464:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575621.526465:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575621.526467:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575621.526468:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575621.526470:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575621.526471:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575621.526473:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575621.526474:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575646 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526479:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575621.526481:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575646 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526485:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575621.526486:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575621.526488:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575621.526491:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575621.526494:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575621.526495:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307225273
00000100:00000001:3.0:1788575621.526498:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307226041
00000100:00000001:3.0:1788575621.526499:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307225273
00000100:00000001:3.0:1788575621.526501:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307226041
00000100:00000001:3.0:1788575621.526503:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575621.526505:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575621.526506:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575621.526510:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575621/real 1788575621]  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575621.526523:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575621.526525:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575621.526526:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575621.526529:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575621.526530:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575621.526532:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575621.526536:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575621.526541:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526544:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575621.526545:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526546:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575621.526547:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575621.526562:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526567:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575621.526569:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526573:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575621.526574:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526575:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575621.526576:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526577:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575621.526578:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575621.526579:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575621.526581:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575621.526582:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575621.526584:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575621.526588:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575621.526591:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526592:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575621.526594:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393223296/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526597:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575621.526598:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575621.526600:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575621.526601:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575621.526602:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575621.526603:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575621.526604:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575621.526606:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:3.0:1788575621.526607:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575621.526608:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575621.526609:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:3.0:1788575621.526610:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575621.526611:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526612:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575621.526614:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393223424/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575646 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575621.526617:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575621.526618:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575621.526619:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575621.526620:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575621.526621:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575621.526621:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575621.526622:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575621.526623:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:3.0:1788575621.526624:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575621.526624:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575621.526625:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:3.0:1788575621.526626:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575621.526627:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526628:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575621.526630:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575621.526631:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575621.526632:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575621.526633:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575621.909652:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575621.909659:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575621.909664:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575621.909666:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575621.909669:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575621.909670:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575621.909729:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575621.909732:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575621.909735:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575622.037504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575622.037511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575622.037514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575622.037519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575622.037523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575622.037527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575622.037529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575622.037532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575622.037535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575622.485492:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307226041, at 4307226048
00000100:00000040:1.0:1788575622.485492:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307226041, at 4307226048
00000100:00000010:1.0:1788575622.485508:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:0.0:1788575622.485511:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00010000:00000001:3.0:1788575622.933630:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575622.933638:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575622.933645:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575622.933646:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575622.933651:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575622.933653:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575622.933722:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575622.933725:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575622.933729:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575623.061532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575623.061543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575623.061546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575623.061551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575623.061566:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575623.061570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575623.061573:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575623.061575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575623.061579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575623.957584:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575623.957590:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575623.957596:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575623.957597:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575623.957601:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575623.957602:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575623.957665:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575623.957670:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575623.957673:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575624.085466:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575624.085472:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575624.085474:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575624.085478:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575624.085481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575624.085484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575624.085486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575624.085488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575624.085490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575624.981509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575624.981515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575624.981520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575624.981523:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575624.981525:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575624.981529:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575624.981531:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575624.981533:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575624.981536:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575624.981538:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575624.981541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575624.981542:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575624.981543:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575624.981545:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575624.981547:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575624.981548:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575624.981549:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575624.981551:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575624.981636:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575624.981639:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575624.981642:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575625.109512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575625.109530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575625.109532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575625.109536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575625.109539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575625.109542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575625.109544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575625.109545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575625.109547:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575626.005512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575626.005518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575626.005523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575626.005524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575626.005528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575626.005529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575626.005612:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575626.005616:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575626.005620:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575626.133606:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575626.133615:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575626.133618:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575626.133625:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575626.133629:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575626.133634:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575626.133636:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575626.133639:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575626.133642:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575626.645515:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575626.645525:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575626.645528:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575626.645532:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575626.645537:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575626.645543:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12557
00000400:00000200:1.0:1788575626.645551:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575626.645557:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575626.645560:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575626.645573:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645577:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575626.645579:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645582:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575626.645585:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575626.645587:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575626.645590:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575626.645593:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575626.645595:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575626.645597:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575626.645600:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645602:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575626.645604:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645606:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575626.645609:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575626.645612:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645614:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575626.645617:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575626.645621:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575626.645623:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575626.645625:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575626.645628:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575626.645637:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788575626.645641:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575626.645644:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575626.645646:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575626.645648:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575626.645650:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575626.645653:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575626.645660:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788575626.645665:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575626.645673:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575626.645682:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575626.645686:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575626.645690:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575626.645709:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575626.645712:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575626.645715:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575626.645717:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575626.645718:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575626.645720:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575626.645723:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12557
00000400:00000200:1.0:1788575626.645727:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575626.645731:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575626.645733:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575626.645736:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645739:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575626.645741:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788575626.645742:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645746:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575626.645748:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1788575626.645749:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575626.645752:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575626.645752:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1788575626.645754:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575626.645758:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575626.645761:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575626.645764:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575626.645764:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575626.645771:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645775:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575626.645780:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645783:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575626.645783:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1788575626.645788:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575626.645788:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575626.645794:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575626.645795:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788575626.645798:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575626.645800:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575626.645803:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575626.645805:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575626.645810:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1788575626.645811:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393223552:192.168.202.110@tcp:250:kworker.0
02000000:00000001:1.0:1788575626.645814:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575626.645820:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575626.645820:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:1.0:1788575626.645826:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575626.645827:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:1.0:1788575626.645831:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575626.645832:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575626.645834:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575626.645837:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:1.0:1788575626.645841:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
02000000:00000010:3.0:1788575626.645845:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
00000020:00000040:1.0:1788575626.645846:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575626.645851:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575626.645851:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575626.645854:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575626.645857:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575626.645858:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575626.645861:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575626.645861:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575626.645868:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
02000000:00000010:1.0:1788575626.645869:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788575626.645874:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575626.645876:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00100000:1.0:1788575626.645881:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575626.645884:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393223552, portal 25
00000100:00000001:3.0:1788575626.645889:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575626.645892:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345478400 : -130483364073216 : ffff8953796f7100)
00000100:00000001:1.0:1788575626.645896:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575626.645900:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575626.645902:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575626.645907:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.645920:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1788575626.645925:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575626.645927:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788575626.645929:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1788575626.645932:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00080000:1.0:1788575626.645934:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575626.645937:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:3.0:1788575626.645938:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393223552, offset 0
00000100:00080000:1.0:1788575626.645939:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575626.645945:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575626.645949:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12557
00000400:00000200:3.0:1788575626.645953:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788575626.645956:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575626.645960:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575626.645961:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575626.645965:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788575626.645966:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645970:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575626.645971:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645973:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575626.645975:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1788575626.645976:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:1.0:1788575626.645977:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575626.645980:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575626.645983:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575626.645985:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575626.645986:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:1.0:1788575626.645987:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575626.645992:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.645995:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575626.645995:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:1.0:1788575626.645998:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575626.646001:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575626.646002:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000040:1.0:1788575626.646005:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575626.646009:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:1.0:1788575626.646010:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575626.646013:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575626.646015:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575626.646018:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575626.646019:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331002:331002:256:4294967295] 192.168.202.110@tcp LPNI seq info [330959:330959:8:4294967295]
02000000:00000001:1.0:1788575626.646021:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575626.646025:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575626.646027:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:3.0:1788575626.646029:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:1.0:1788575626.646030:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575626.646033:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575626.646036:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788575626.646038:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575626.646040:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575626.646042:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575626.646044:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575626.646046:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575626.646049:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575626.646051:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788575626.646054:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575626.646057:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575626.646057:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1788575626.646068:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575626.646071:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575626.646073:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788575626.646076:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1788575626.646080:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000100:00000001:1.0:1788575626.646081:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575626.646086:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575626.646088:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:3.0:1788575626.646093:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788575626.646094:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12567)
00000800:00000200:3.0:1788575626.646099:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788575626.646106:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575626.646110:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575626.646115:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575626.646117:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575626.646123:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575626.646127:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575626.646133:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575626.646136:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575626.646148:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575626.646153:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646162:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575626.646166:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575651 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646173:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575626.646175:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575626.646180:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575626.646185:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575626.646188:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307231161
00000100:00000010:3.0:1788575626.646193:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575626.646199:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575626.646214:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307231161
00000100:00000010:3.0:1788575626.646218:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575626.646222:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575626.646225:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575626.646228:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575626.646230:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575626.646234:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575626/real 1788575626]  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575626.646256:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575626.646257:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575626.646259:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575626.646262:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575626.646264:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575626.646266:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575626.646270:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575626.646279:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646286:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575626.646287:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646290:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575626.646292:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575626.646296:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646303:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575626.646308:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646314:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575626.646317:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646319:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575626.646320:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646323:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575626.646325:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575626.646328:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575626.646331:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575626.646333:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575626.646337:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575626.646346:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393223552:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575626.646350:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646352:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575626.646355:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7100 x1875444393223552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646361:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575626.646363:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575626.646366:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575626.646369:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575626.646371:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575626.646372:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575626.646374:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575626.646377:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:3.0:1788575626.646380:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575626.646382:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575626.646384:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7100.
00000100:00000001:3.0:1788575626.646387:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575626.646389:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646409:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646417:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575626.646419:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575626.646421:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575626.646423:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575626.646424:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575626.646428:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646435:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575626.646436:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575626.646439:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575626.646441:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575626.646443:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575626.646445:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575626.646449:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575626.646451:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575626.646453:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575626.646454:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575626.646457:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575626.646459:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575626.646463:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575626.646466:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575626.646469:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575626.646472:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393223680, portal 10
00000100:00000001:3.0:1788575626.646473:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575626.646474:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345481088 : -130483364070528 : ffff8953796f7b80)
00000100:00000040:3.0:1788575626.646481:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646487:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575626.646489:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575626.646491:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575626.646494:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393223680, offset 0
00000400:00000200:3.0:1788575626.646497:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575626.646501:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575626.646505:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575626.646508:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575626.646511:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575626.646513:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575626.646515:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575626.646517:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331003:331003:256:4294967295] 192.168.202.110@tcp LPNI seq info [330960:330960:8:4294967295]
00000400:00000200:3.0:1788575626.646521:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575626.646527:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575626.646532:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575626.646535:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575626.646539:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575626.646542:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788575626.646546:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575626.646548:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575626.646552:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575626.646553:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575626.646556:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575626.646559:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575626.646562:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575626.646576:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575626.646579:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575626.646582:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646590:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575626.646594:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575651 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646600:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575626.646602:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575626.646606:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575626.646609:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575626.646612:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575626.646614:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307230247
00000100:00000001:3.0:1788575626.646617:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307231161
00000100:00000001:3.0:1788575626.646619:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307230247
00000100:00000001:3.0:1788575626.646621:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307231161
00000100:00000001:3.0:1788575626.646623:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575626.646625:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575626.646627:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575626.646630:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575626/real 1788575626]  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575626.646639:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575626.646640:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575626.646642:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575626.646645:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575626.646646:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575626.646649:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575626.646652:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575626.646660:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646666:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575626.646668:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646669:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575626.646670:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575626.646674:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646680:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575626.646684:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646690:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575626.646692:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646694:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575626.646695:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646697:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575626.646699:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575626.646702:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575626.646705:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575626.646707:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575626.646711:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575626.646718:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575626.646722:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575626.646726:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646733:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575626.646735:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575626.646737:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575626.646738:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575626.646740:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575626.646744:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223808:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575626.646747:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575626.646749:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575626.646750:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575626.646752:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575626.646755:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575626.646758:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575626.646761:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575626.646764:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575626.646767:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575626.646770:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393223808, portal 4
00000100:00000001:3.0:1788575626.646771:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575626.646773:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345469440 : -130483364082176 : ffff8953796f4e00)
00000100:00000040:3.0:1788575626.646779:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646786:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575626.646788:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575626.646790:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000200:3.0:1788575626.646792:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393223808, offset 0
00000400:00000200:3.0:1788575626.646796:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575626.646800:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575626.646804:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575626.646806:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575626.646809:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575626.646811:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575626.646814:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575626.646817:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331004:331004:256:4294967295] 192.168.202.110@tcp LPNI seq info [330961:330961:8:4294967295]
00000400:00000200:3.0:1788575626.646821:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575626.646828:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575626.646833:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575626.646835:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000800:00000200:3.0:1788575626.646839:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575626.646842:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788575626.646846:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575626.646848:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575626.646852:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575626.646854:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:3.0:1788575626.646857:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575626.646860:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575626.646863:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:3.0:1788575626.646865:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:3.0:1788575626.646867:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575626.646871:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575651 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646878:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575626.646881:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575651 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646888:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575626.646890:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575626.646893:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575626.646895:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575626.646899:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575626.646900:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307230393
00000100:00000001:3.0:1788575626.646903:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307231161
00000100:00000001:3.0:1788575626.646905:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307230393
00000100:00000001:3.0:1788575626.646908:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307231161
00000100:00000001:3.0:1788575626.646910:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575626.646912:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575626.646914:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575626.646917:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575626/real 1788575626]  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575626.646924:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575626.646925:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575626.646927:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575626.646929:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575626.646931:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575626.646932:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575626.646935:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575626.646944:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646950:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575626.646952:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646953:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575626.646955:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575626.646958:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646965:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575626.646969:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.646976:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575626.646978:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646980:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575626.646981:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.646983:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575626.646985:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575626.646988:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575626.646991:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575626.646993:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575626.646997:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575626.647006:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393223808:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575626.647009:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.647011:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575626.647014:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7b80 x1875444393223680/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.647021:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575626.647023:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575626.647025:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575626.647028:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575626.647029:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575626.647031:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575626.647033:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575626.647034:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:3.0:1788575626.647037:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575626.647038:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575626.647039:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7b80.
00000100:00000001:3.0:1788575626.647042:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575626.647043:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.647045:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575626.647048:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4e00 x1875444393223808/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575651 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575626.647054:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575626.647056:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575626.647057:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575626.647060:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575626.647061:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575626.647062:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575626.647064:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575626.647065:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:3.0:1788575626.647067:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575626.647068:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575626.647070:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4e00.
00000100:00000001:3.0:1788575626.647072:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575626.647073:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.647075:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575626.647078:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575626.647079:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575626.647081:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575626.647082:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575627.029501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575627.029509:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575627.029511:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575627.029514:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575627.029518:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575627.029520:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575627.029522:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575627.029524:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575627.029528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575627.029530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575627.029533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575627.029534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575627.029624:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575627.029627:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575627.029630:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575627.157513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575627.157520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575627.157522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575627.157527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575627.157531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575627.157534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575627.157537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575627.157540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575627.157543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575627.605495:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307231161, at 4307231168
00000100:00000040:1.0:1788575627.605495:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307231161, at 4307231168
00000100:00000010:1.0:1788575627.605507:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575627.605509:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575628.053497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575628.053503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575628.053507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575628.053509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575628.053512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575628.053513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575628.053601:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575628.053605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575628.053608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575628.181523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575628.181530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575628.181532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575628.181538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575628.181542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575628.181545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575628.181549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575628.181551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575628.181554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575629.077467:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575629.077488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575629.077494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575629.077495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575629.077498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575629.077500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575629.077553:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575629.077557:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575629.077560:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575629.205499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575629.205507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575629.205509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575629.205514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575629.205517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575629.205521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575629.205524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575629.205526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575629.205529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575630.101522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575630.101529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575630.101535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575630.101536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575630.101539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575630.101540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575630.101617:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575630.101620:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575630.101624:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575630.229501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575630.229510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575630.229512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575630.229517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575630.229521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575630.229524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575630.229527:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575630.229529:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575630.229532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575631.125498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575631.125505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575631.125510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575631.125519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575631.125523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575631.125524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575631.125593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575631.125596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575631.125599:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575631.253495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575631.253502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575631.253504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575631.253509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575631.253517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575631.253521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.253524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575631.253526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.253528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575631.765516:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575631.765526:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575631.765529:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575631.765532:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575631.765537:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575631.765544:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12562
00000400:00000200:3.0:1788575631.765553:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575631.765559:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575631.765562:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575631.765565:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765568:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575631.765571:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765573:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575631.765577:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575631.765591:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575631.765594:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575631.765598:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575631.765600:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575631.765602:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575631.765605:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765608:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575631.765610:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765612:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575631.765614:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575631.765617:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765626:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575631.765628:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575631.765631:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575631.765634:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575631.765636:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575631.765639:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575631.765648:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788575631.765651:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575631.765654:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575631.765656:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575631.765659:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575631.765662:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575631.765665:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575631.765672:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788575631.765678:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575631.765686:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575631.765696:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575631.765700:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575631.765704:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575631.765729:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575631.765733:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575631.765736:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575631.765737:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575631.765739:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575631.765741:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575631.765744:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12562
00000400:00000200:3.0:1788575631.765748:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575631.765751:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575631.765753:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575631.765757:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765759:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575631.765762:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765764:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575631.765767:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575631.765767:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575631.765771:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575631.765772:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575631.765774:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575631.765776:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788575631.765778:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575631.765782:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575631.765785:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575631.765786:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575631.765790:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765795:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575631.765799:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765804:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575631.765806:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:3.0:1788575631.765808:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575631.765810:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575631.765814:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575631.765815:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575631.765820:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575631.765820:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575631.765824:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575631.765825:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575631.765831:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1788575631.765832:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393223936:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575631.765837:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.765840:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575631.765840:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575631.765845:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:3.0:1788575631.765846:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575631.765849:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788575631.765851:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575631.765852:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575631.765855:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575631.765859:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
00000100:00000010:3.0:1788575631.765861:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
02000000:00000001:2.0:1788575631.765863:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575631.765865:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575631.765870:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575631.765871:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575631.765873:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575631.765876:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1788575631.765877:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
02000000:00000001:3.0:1788575631.765878:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575631.765882:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575631.765886:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
02000000:00000010:3.0:1788575631.765888:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575631.765893:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575631.765895:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393223936, portal 25
00000100:00000001:2.0:1788575631.765898:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575631.765900:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00100000:3.0:1788575631.765901:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788575631.765908:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575631.765914:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575631.765918:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.765919:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575631.765923:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575631.765926:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575631.765928:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575631.765933:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393223936, offset 0
00000400:00000200:2.0:1788575631.765942:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575631.765942:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575631.765947:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575631.765952:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575631.765953:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575631.765956:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575631.765958:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575631.765959:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788575631.765963:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575631.765968:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1788575631.765968:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12562
00000400:00000200:2.0:1788575631.765976:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575631.765980:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575631.765983:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575631.765987:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331005:331005:256:4294967295] 192.168.202.110@tcp LPNI seq info [330962:330962:8:4294967295]
00000400:00000200:2.0:1788575631.765992:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575631.765998:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575631.766006:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575631.766009:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1788575631.766010:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575631.766017:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575631.766021:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1788575631.766022:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575631.766026:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575631.766028:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000100:00000001:3.0:1788575631.766029:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575631.766033:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575631.766036:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:2.0:1788575631.766037:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788575631.766039:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.0:1788575631.766041:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000040:3.0:1788575631.766043:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575631.766048:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.1:1788575631.766048:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575631.766052:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:3.0:1788575631.766054:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000100:2.0:1788575631.766059:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575631.766061:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575631.766062:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000100:00000001:3.0:1788575631.766066:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575631.766069:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575631.766070:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575631.766076:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575631.766081:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000100:00000001:3.0:1788575631.766081:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:2.0:1788575631.766085:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000020:00000040:3.0:1788575631.766087:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788575631.766090:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00080000:3.0:1788575631.766093:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:2.0:1788575631.766097:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575631.766098:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575631.766103:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575631.766107:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575631.766110:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1788575631.766110:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575631.766114:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1788575631.766115:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1788575631.766118:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575631.766123:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766125:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575631.766127:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766128:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000010:3.0:1788575631.766132:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575631.766134:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575631.766136:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575631.766141:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575631.766142:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575631.766144:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307236281
02000000:00000001:3.0:1788575631.766145:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575631.766149:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:2.0:1788575631.766150:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c6c0.
02000000:00000001:3.0:1788575631.766151:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575631.766157:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:3.0:1788575631.766161:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766163:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307236281
00000100:00000010:2.0:1788575631.766168:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ca80.
02000000:00000010:3.0:1788575631.766168:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:2.0:1788575631.766174:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575631.766175:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766176:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575631.766179:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766181:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575631.766184:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:2.0:1788575631.766188:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575631/real 1788575631]  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575631.766206:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000001:2.0:1788575631.766209:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1788575631.766209:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1788575631.766212:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575631.766215:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000100:00000040:3.0:1788575631.766216:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575631.766218:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575631.766221:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575631.766225:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575631.766230:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575631.766232:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575631.766238:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1788575631.766243:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575631.766244:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575631.766253:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:3.0:1788575631.766254:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12573)
00000100:00000001:2.0:1788575631.766256:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766259:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575631.766261:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575631.766265:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766272:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575631.766278:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766285:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575631.766288:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766289:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575631.766291:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766293:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575631.766295:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575631.766298:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575631.766301:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575631.766304:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575631.766307:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575631.766313:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393223936:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575631.766316:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766317:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575631.766319:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393223936/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766323:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575631.766324:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575631.766327:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575631.766328:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575631.766330:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575631.766331:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575631.766333:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575631.766334:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575631.766336:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575631.766338:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575631.766339:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575631.766341:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575631.766342:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766343:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766348:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575631.766349:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766351:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575631.766351:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575631.766352:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575631.766355:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766359:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575631.766360:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575631.766361:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575631.766362:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575631.766363:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575631.766365:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224064:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575631.766367:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575631.766368:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575631.766369:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575631.766370:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575631.766372:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575631.766373:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575631.766375:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575631.766377:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575631.766379:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575631.766381:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393224064, portal 10
00000100:00000001:2.0:1788575631.766383:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575631.766384:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:2.0:1788575631.766388:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766411:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575631.766413:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575631.766415:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575631.766416:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393224064, offset 0
00000400:00000200:2.0:1788575631.766418:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575631.766421:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575631.766423:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575631.766425:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575631.766427:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575631.766429:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575631.766430:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575631.766432:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331006:331006:256:4294967295] 192.168.202.110@tcp LPNI seq info [330963:330963:8:4294967295]
00000400:00000200:2.0:1788575631.766434:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575631.766438:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575631.766441:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575631.766443:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575631.766445:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575631.766447:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788575631.766449:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575631.766451:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:2.0:1788575631.766453:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575631.766454:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575631.766456:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575631.766458:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575631.766460:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575631.766461:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575631.766462:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575631.766465:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766473:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575631.766483:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766487:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766488:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575631.766490:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575631.766492:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575631.766494:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575631.766496:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307235367
00000100:00000001:2.0:1788575631.766497:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307236281
00000100:00000001:2.0:1788575631.766499:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307235367
00000100:00000001:2.0:1788575631.766500:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307236281
00000100:00000001:2.0:1788575631.766501:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575631.766503:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766504:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575631.766506:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575631/real 1788575631]  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575631.766510:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575631.766511:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575631.766512:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575631.766514:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575631.766515:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575631.766516:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575631.766517:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575631.766522:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766526:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575631.766527:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766528:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575631.766529:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575631.766535:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766540:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575631.766542:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766547:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575631.766548:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766549:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575631.766550:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766551:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575631.766552:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575631.766554:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575631.766556:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575631.766557:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575631.766559:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575631.766563:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224064:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575631.766565:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575631.766567:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766571:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575631.766571:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575631.766573:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575631.766574:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575631.766575:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575631.766577:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224192:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575631.766592:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575631.766593:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575631.766594:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575631.766595:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575631.766598:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575631.766599:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575631.766602:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575631.766603:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575631.766605:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575631.766607:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393224192, portal 4
00000100:00000001:2.0:1788575631.766608:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575631.766609:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788575631.766613:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766617:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575631.766618:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575631.766620:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575631.766621:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393224192, offset 0
00000400:00000200:2.0:1788575631.766623:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575631.766625:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575631.766627:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575631.766629:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575631.766631:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575631.766632:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575631.766634:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575631.766635:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331007:331007:256:4294967295] 192.168.202.110@tcp LPNI seq info [330964:330964:8:4294967295]
00000400:00000200:2.0:1788575631.766638:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575631.766641:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575631.766643:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575631.766645:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575631.766647:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575631.766649:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1788575631.766651:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575631.766653:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:2.0:1788575631.766656:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575631.766657:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575631.766659:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575631.766661:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575631.766663:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575631.766664:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575631.766665:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575631.766667:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766672:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575631.766673:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766677:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766678:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575631.766680:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575631.766681:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575631.766683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575631.766684:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307235513
00000100:00000001:2.0:1788575631.766686:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307236281
00000100:00000001:2.0:1788575631.766687:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307235513
00000100:00000001:2.0:1788575631.766688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307236281
00000100:00000001:2.0:1788575631.766689:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575631.766690:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766692:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575631.766693:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575631/real 1788575631]  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575631.766697:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575631.766699:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575631.766700:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575631.766702:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575631.766704:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575631.766706:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575631.766708:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575631.766716:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766722:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575631.766723:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766725:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575631.766727:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575631.766731:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766737:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575631.766740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766746:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575631.766748:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766750:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575631.766751:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766753:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575631.766755:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575631.766757:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575631.766760:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575631.766761:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575631.766765:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575631.766776:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224192:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575631.766780:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766781:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575631.766784:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393224064/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766790:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575631.766791:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575631.766793:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575631.766795:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575631.766796:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575631.766799:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575631.766801:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575631.766803:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575631.766805:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575631.766806:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575631.766807:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575631.766809:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575631.766810:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766812:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575631.766814:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393224192/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575631.766819:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575631.766820:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575631.766822:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575631.766824:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575631.766825:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575631.766826:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575631.766827:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575631.766828:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575631.766830:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575631.766830:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575631.766832:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575631.766834:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575631.766835:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766837:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575631.766840:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575631.766841:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575631.766842:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575631.766844:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575632.149509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575632.149514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575632.149517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575632.149518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575632.149520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575632.149521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575632.149594:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575632.149598:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575632.149601:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575632.277480:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575632.277487:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575632.277490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575632.277495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575632.277498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575632.277502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575632.277504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575632.277507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575632.277510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575632.725522:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307236281, at 4307236288
00000100:00000010:0.0:1788575632.725537:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00000100:00000040:1.0:1788575632.725672:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307236281, at 4307236288
00000100:00000010:1.0:1788575632.725678:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00010000:00000001:3.0:1788575633.173490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575633.173496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575633.173502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575633.173503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575633.173506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575633.173508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575633.173562:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575633.173566:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575633.173569:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575633.301514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575633.301521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575633.301523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575633.301537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575633.301541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575633.301545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575633.301547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575633.301550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575633.301553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575633.813523:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1788575633.813532:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1788575633.813536:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575633.813729:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575633.813741:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575633.813742:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575633.813749:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575633.813752:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575634.197512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575634.197519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575634.197523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575634.197525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575634.197528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575634.197529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575634.197655:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575634.197658:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575634.197661:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575634.325507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575634.325514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575634.325517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575634.325522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575634.325526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575634.325530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575634.325532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575634.325534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575634.325537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575635.221542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575635.221549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575635.221555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575635.221558:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575635.221560:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575635.221564:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575635.221567:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575635.221569:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575635.221571:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575635.221573:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575635.221576:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575635.221579:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575635.221580:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575635.221581:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575635.221583:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575635.221584:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575635.221585:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575635.221587:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575635.221711:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575635.221714:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575635.221718:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575635.349527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575635.349534:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575635.349536:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575635.349541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575635.349545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575635.349548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575635.349551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575635.349554:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575635.349557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575636.245690:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575636.245697:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575636.245703:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575636.245704:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575636.245708:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575636.245709:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575636.245875:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575636.245878:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575636.245881:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575636.373468:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575636.373475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575636.373477:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575636.373483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575636.373486:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575636.373490:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.373493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575636.373495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.373498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575636.885512:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575636.885522:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575636.885525:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575636.885529:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575636.885533:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575636.885541:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12568
00000400:00000200:0.0:1788575636.885549:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575636.885555:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575636.885558:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575636.885562:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575636.885564:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575636.885567:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575636.885569:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575636.885573:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575636.885575:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575636.885577:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575636.885581:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575636.885583:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575636.885585:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575636.885589:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575636.885590:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575636.885592:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575636.885610:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575636.885614:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575636.885618:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575636.885621:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575636.885624:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575636.885627:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575636.885630:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575636.885632:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575636.885634:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575636.885644:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788575636.885647:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575636.885651:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575636.885653:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575636.885655:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575636.885658:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575636.885662:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575636.885670:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575636.885676:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575636.885685:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575636.885696:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575636.885699:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575636.885704:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575636.885726:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575636.885729:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575636.885732:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575636.885734:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575636.885735:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575636.885738:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575636.885741:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12568
00000400:00000200:0.0:1788575636.885745:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575636.885749:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575636.885752:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575636.885756:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575636.885758:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575636.885760:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575636.885763:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575636.885764:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788575636.885765:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575636.885769:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575636.885769:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788575636.885771:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575636.885773:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1788575636.885775:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575636.885779:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575636.885782:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1788575636.885783:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575636.885792:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575636.885797:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575636.885799:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1788575636.885801:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575636.885803:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575636.885804:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575636.885809:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575636.885810:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1788575636.885813:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575636.885815:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575636.885817:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575636.885820:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575636.885821:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1788575636.885827:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393224320:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575636.885831:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575636.885835:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575636.885836:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575636.885837:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575636.885841:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575636.885841:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788575636.885845:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:2.0:1788575636.885847:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
00000008:00100000:0.0:1788575636.885850:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575636.885850:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788575636.885856:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575636.885857:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575636.885862:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575636.885863:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00000010:0.0:1788575636.885869:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000400:00000010:2.0:1788575636.885871:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000020:00000040:0.0:1788575636.885874:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000200:2.0:1788575636.885877:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393224320, portal 25
02000000:00000001:0.0:1788575636.885879:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575636.885881:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1788575636.885882:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575636.885883:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000001:0.0:1788575636.885888:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575636.885892:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:2.0:1788575636.885892:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575636.885895:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575636.885903:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788575636.885907:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575636.885911:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:0.0:1788575636.885915:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788575636.885918:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575636.885924:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000001:0.0:1788575636.885929:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1788575636.885931:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393224320, offset 0
00000100:00000001:0.0:1788575636.885932:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575636.885939:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575636.885941:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575636.885951:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575636.885953:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575636.885957:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575636.885960:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:0.0:1788575636.885965:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:2.0:1788575636.885968:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575636.885969:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575636.885971:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575636.885974:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575636.885978:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12568
00000400:00000200:2.0:1788575636.885981:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575636.885986:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575636.885987:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575636.885992:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575636.885995:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575636.885996:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331008:331008:256:4294967295] 192.168.202.110@tcp LPNI seq info [330965:330965:8:4294967295]
00000100:00080000:0.0:1788575636.886001:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575636.886003:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788575636.886009:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575636.886012:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575636.886019:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575636.886019:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788575636.886024:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575636.886029:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.0:1788575636.886029:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:0.0:1788575636.886033:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575636.886035:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000010:2.0:1788575636.886035:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000020:00000040:0.0:1788575636.886039:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575636.886043:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.0:1788575636.886043:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575636.886047:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:0.0:1788575636.886048:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.1:1788575636.886052:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575636.886056:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000040:0.0:1788575636.886057:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000100:2.0:1788575636.886062:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575636.886063:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000010:2.0:1788575636.886065:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000100:00000040:0.0:1788575636.886069:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575636.886071:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1788575636.886073:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575636.886077:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:0.0:1788575636.886079:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575636.886082:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575636.886086:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575636.886090:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00080000:0.0:1788575636.886096:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575636.886100:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575636.886101:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575636.886104:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575636.886108:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575636.886113:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575636.886116:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575636.886121:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575636.886123:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000008:00000001:0.0:1788575636.886124:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575636.886129:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575636.886130:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1788575636.886134:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788575636.886140:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:2.0:1788575636.886141:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575636.886143:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575636.886145:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:0.0:1788575636.886146:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575636.886150:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575636.886151:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575636.886152:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575636.886153:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575636.886159:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1788575636.886160:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:2.0:1788575636.886162:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307241401
00000100:00000001:0.0:1788575636.886165:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575636.886169:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c2a0.
00000100:00100000:0.0:1788575636.886174:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575636.886178:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575636.886185:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307241401
00000100:00000001:0.0:1788575636.886186:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575636.886192:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575636.886192:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c300.
00000100:00000040:0.0:1788575636.886199:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575636.886207:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575636.886211:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:0.0:1788575636.886212:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575636.886217:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1788575636.886218:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575636.886223:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575636.886226:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12578)
00000100:00000001:2.0:1788575636.886227:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575636.886236:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575636/real 1788575636]  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575636.886245:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575636.886247:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575636.886249:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575636.886252:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575636.886254:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575636.886256:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575636.886259:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575636.886267:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886273:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575636.886275:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886278:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575636.886280:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575636.886284:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886288:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575636.886291:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886296:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575636.886298:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886299:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575636.886300:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886301:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575636.886302:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575636.886304:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575636.886306:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575636.886308:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575636.886310:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575636.886315:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393224320:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575636.886318:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886325:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575636.886327:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393224320/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886334:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575636.886336:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575636.886339:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575636.886341:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575636.886342:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575636.886344:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575636.886345:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575636.886347:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788575636.886350:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575636.886351:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575636.886353:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:2.0:1788575636.886356:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575636.886357:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886359:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886365:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575636.886367:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575636.886369:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575636.886370:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575636.886372:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575636.886375:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886382:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575636.886383:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575636.886385:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575636.886387:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575636.886389:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575636.886419:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575636.886422:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575636.886423:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575636.886425:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575636.886426:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575636.886428:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575636.886429:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575636.886432:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575636.886434:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575636.886436:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575636.886438:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393224448, portal 10
00000100:00000001:2.0:1788575636.886440:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575636.886441:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00000040:2.0:1788575636.886446:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886452:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575636.886454:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575636.886457:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575636.886459:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393224448, offset 0
00000400:00000200:2.0:1788575636.886463:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575636.886466:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575636.886470:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575636.886473:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575636.886475:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575636.886477:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575636.886480:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575636.886483:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331009:331009:256:4294967295] 192.168.202.110@tcp LPNI seq info [330966:330966:8:4294967295]
00000400:00000200:2.0:1788575636.886486:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575636.886492:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575636.886497:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575636.886500:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575636.886504:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575636.886507:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575636.886510:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575636.886513:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1788575636.886516:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575636.886518:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575636.886520:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575636.886524:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575636.886527:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575636.886529:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575636.886531:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575636.886534:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886547:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575636.886549:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886553:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575636.886553:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575636.886556:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575636.886558:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575636.886561:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575636.886562:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307240487
00000100:00000001:2.0:1788575636.886564:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307241401
00000100:00000001:2.0:1788575636.886565:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307240487
00000100:00000001:2.0:1788575636.886567:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307241401
00000100:00000001:2.0:1788575636.886568:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575636.886569:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575636.886571:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575636.886573:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575636/real 1788575636]  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575636.886577:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575636.886578:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575636.886579:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575636.886580:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575636.886581:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575636.886582:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575636.886584:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575636.886588:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886592:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575636.886607:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886610:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575636.886612:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575636.886615:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886622:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575636.886625:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886631:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575636.886633:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886635:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575636.886636:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886638:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575636.886640:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575636.886642:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575636.886645:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575636.886648:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575636.886652:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575636.886660:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575636.886663:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575636.886667:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886673:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575636.886674:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575636.886676:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575636.886677:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575636.886679:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575636.886689:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575636.886692:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575636.886694:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575636.886696:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575636.886698:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575636.886701:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575636.886703:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575636.886707:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575636.886709:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575636.886713:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575636.886716:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393224576, portal 4
00000100:00000001:2.0:1788575636.886717:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575636.886718:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:2.0:1788575636.886725:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886731:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575636.886733:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575636.886735:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575636.886738:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393224576, offset 0
00000400:00000200:2.0:1788575636.886742:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575636.886746:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575636.886750:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575636.886752:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575636.886755:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575636.886757:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575636.886759:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575636.886761:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331010:331010:256:4294967295] 192.168.202.110@tcp LPNI seq info [330967:330967:8:4294967295]
00000400:00000200:2.0:1788575636.886764:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575636.886768:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575636.886771:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575636.886773:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575636.886776:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575636.886778:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575636.886780:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575636.886783:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1788575636.886786:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575636.886788:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575636.886791:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575636.886793:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575636.886796:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575636.886798:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575636.886800:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575636.886803:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886811:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575636.886814:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886820:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575636.886822:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575636.886825:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575636.886828:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575636.886831:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575636.886832:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307240633
00000100:00000001:2.0:1788575636.886834:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307241401
00000100:00000001:2.0:1788575636.886836:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307240633
00000100:00000001:2.0:1788575636.886837:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307241401
00000100:00000001:2.0:1788575636.886838:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575636.886839:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575636.886840:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575636.886843:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575636/real 1788575636]  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575636.886847:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575636.886848:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575636.886849:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575636.886851:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575636.886852:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575636.886853:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575636.886855:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575636.886860:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886863:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575636.886864:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886865:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575636.886866:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575636.886868:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886871:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575636.886873:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886877:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575636.886878:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886879:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575636.886880:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886881:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575636.886882:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575636.886883:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575636.886885:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575636.886886:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575636.886890:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575636.886903:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575636.886907:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886908:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575636.886911:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393224448/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886917:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575636.886919:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575636.886921:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575636.886923:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575636.886925:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575636.886926:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575636.886928:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575636.886930:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788575636.886933:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575636.886934:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575636.886935:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:2.0:1788575636.886937:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575636.886938:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886939:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575636.886942:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393224576/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575636.886946:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575636.886946:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575636.886948:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575636.886949:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575636.886950:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575636.886951:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575636.886953:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575636.886954:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:2.0:1788575636.886956:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575636.886957:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575636.886958:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:2.0:1788575636.886961:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575636.886962:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886964:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575636.886968:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575636.886969:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575636.886971:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575636.886972:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575637.269517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575637.269525:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575637.269527:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575637.269530:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575637.269541:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575637.269543:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575637.269545:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575637.269547:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575637.269551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575637.269552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575637.269555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575637.269556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575637.269624:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575637.269628:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575637.269631:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575637.397541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575637.397548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575637.397550:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575637.397555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575637.397558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575637.397562:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575637.397564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575637.397567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575637.397570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575637.845526:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307241401, at 4307241408
00000100:00000010:0.0:1788575637.845542:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000100:00000040:1.0:1788575637.845634:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307241401, at 4307241408
00000100:00000010:1.0:1788575637.845640:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00010000:00000001:3.0:1788575638.293525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575638.293532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575638.293537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575638.293539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575638.293552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575638.293554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575638.293728:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575638.293732:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575638.293735:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575638.421718:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575638.421723:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575638.421725:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575638.421730:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575638.421733:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575638.421736:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575638.421739:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575638.421741:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575638.421743:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575639.317514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575639.317520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575639.317525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575639.317526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575639.317529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575639.317530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575639.317590:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575639.317593:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575639.317595:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575639.445532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575639.445537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575639.445539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575639.445543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575639.445547:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575639.445551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575639.445553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575639.445556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575639.445559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575640.341519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575640.341526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575640.341531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575640.341532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575640.341536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575640.341537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575640.341601:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575640.341636:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575640.341639:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575640.469526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575640.469535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575640.469537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575640.469548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575640.469552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575640.469556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575640.469559:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575640.469562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575640.469564:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575641.365509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575641.365516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575641.365521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575641.365523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575641.365526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575641.365527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575641.365592:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575641.365596:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575641.365599:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575641.493589:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575641.493604:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575641.493618:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575641.493625:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575641.493628:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575641.493632:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575641.493635:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575641.493638:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575641.493642:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575642.005719:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575642.005729:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575642.005731:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575642.005735:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575642.005739:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575642.005747:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12573
00000400:00000200:2.0:1788575642.005755:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575642.005761:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575642.005765:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575642.005768:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.005771:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575642.005774:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.005776:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575642.005780:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575642.005783:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575642.005785:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575642.005789:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575642.005791:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575642.005792:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575642.005795:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.005805:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575642.005806:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.005809:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575642.005813:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575642.005816:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575642.005818:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575642.005821:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575642.005825:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575642.005828:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575642.005830:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575642.005832:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575642.005843:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01f80.
00000020:00000040:2.0:1788575642.005847:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575642.005850:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575642.005852:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575642.005855:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575642.005858:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575642.005863:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575642.005872:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12c00.
00000100:00000001:2.0:1788575642.005879:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575642.005888:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575642.005899:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575642.005903:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575642.005908:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8c8ec7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575642.005939:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575642.005943:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575642.005946:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575642.005948:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575642.005950:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575642.005952:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575642.005956:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12573
00000400:00000200:2.0:1788575642.005960:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575642.005963:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575642.005965:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575642.005968:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.005970:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575642.005973:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.005975:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575642.005977:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575642.005979:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575642.005980:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575642.005981:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575642.005985:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575642.005987:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1788575642.005988:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575642.005992:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575642.005992:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575642.006000:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.006003:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575642.006003:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575642.006007:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.006011:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575642.006016:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575642.006020:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1788575642.006021:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575642.006024:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575642.006024:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575642.006028:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575642.006030:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575642.006034:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575642.006035:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575642.006039:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575642.006040:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575642.006043:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:3.0:1788575642.006045:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393224704:192.168.202.110@tcp:250:kworker.0
00000008:00100000:2.0:1788575642.006048:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575642.006052:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1788575642.006053:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575642.006056:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575642.006058:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1788575642.006060:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03800.
02000000:00000001:3.0:1788575642.006062:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575642.006064:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575642.006066:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1788575642.006068:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575642.006071:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575642.006073:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
00000100:00000001:2.0:1788575642.006074:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575642.006076:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575642.006078:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575642.006080:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575642.006086:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000400:00000200:3.0:1788575642.006086:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575642.006093:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575642.006094:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00100000:2.0:1788575642.006099:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575642.006102:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575642.006107:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575642.006109:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575642.006115:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014d98894] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575642.006118:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393224704, portal 25
00000100:00000001:3.0:1788575642.006123:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575642.006126:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000001:2.0:1788575642.006127:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575642.006132:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575642.006136:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575642.006137:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575642.006140:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575642.006142:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575642.006148:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:3.0:1788575642.006152:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1788575642.006153:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12573
00000400:00000200:2.0:1788575642.006159:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575642.006162:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1788575642.006165:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575642.006166:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575642.006171:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:3.0:1788575642.006171:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526770.
00000100:00000001:2.0:1788575642.006176:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1788575642.006178:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393224704, offset 0
00000100:00000040:2.0:1788575642.006179:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.006183:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575642.006185:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575642.006188:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:3.0:1788575642.006189:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:2.0:1788575642.006190:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575642.006193:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575642.006195:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575642.006198:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575642.006201:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575642.006204:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.006207:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575642.006209:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575642.006211:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575642.006217:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788575642.006222:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575642.006223:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:2.0:1788575642.006226:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575642.006230:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1788575642.006232:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575642.006236:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575642.006237:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575642.006240:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575642.006243:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331011:331011:256:4294967295] 192.168.202.110@tcp LPNI seq info [330968:330968:8:4294967295]
02000000:00000001:2.0:1788575642.006245:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575642.006250:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575642.006251:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575642.006254:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575642.006257:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575642.006260:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575642.006264:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575642.006267:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000010:2.0:1788575642.006270:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02a00.
00000020:00000040:2.0:1788575642.006274:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:3.0:1788575642.006275:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788575642.006280:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575642.006283:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575642.006283:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bb00.
00000100:00000001:2.0:1788575642.006286:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575642.006287:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575642.006288:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575642.006294:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788575642.006298:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575642.006300:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:2.0:1788575642.006305:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1788575642.006308:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575642.006317:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575642.006320:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575642.006324:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cce910e4] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575642.006331:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575642.006332:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575642.006334:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575642.006339:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575642.006340:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575642.006344:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307246521
00000100:00000010:3.0:1788575642.006351:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000040:2.0:1788575642.006355:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12583)
00000100:00000001:3.0:1788575642.006358:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575642.006365:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307246521
00000100:00000010:3.0:1788575642.006369:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575642.006372:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575642.006376:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575642.006379:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575642.006382:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575642.006385:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575642.006416:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575642.006419:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1788575642.006421:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575642.006423:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575642.006424:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575642.006428:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575642.006435:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575642.006437:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575642.006440:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575642.006441:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575642.006443:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575642.006446:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224832:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575642.006450:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575642.006452:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575642.006453:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575642.006455:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575642.006459:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575642.006461:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575642.006465:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575642.006468:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c52c0.
00000400:00000010:3.0:1788575642.006472:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526440.
00000100:00000200:3.0:1788575642.006475:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393224832, portal 10
00000100:00000001:3.0:1788575642.006477:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575642.006478:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:3.0:1788575642.006484:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575642.006490:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575642.006492:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575642.006495:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526b28.
00000100:00000200:3.0:1788575642.006497:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393224832, offset 0
00000400:00000200:3.0:1788575642.006503:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575642.006506:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575642.006509:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575642.006513:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575642.006515:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575642.006517:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575642.006519:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575642.006522:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331012:331012:255:4294967295] 192.168.202.110@tcp LPNI seq info [330969:330969:7:4294967295]
00000400:00000200:3.0:1788575642.006526:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575642.006532:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575642.006536:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575642.006539:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728bd00.
00000800:00000200:3.0:1788575642.006543:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575642.006560:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575642.006563:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575642.006567:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575642.006570:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575642.006572:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307245607
00000100:00000001:3.0:1788575642.006574:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307246521
00000100:00000001:3.0:1788575642.006576:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307245607
00000100:00000001:3.0:1788575642.006578:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307246521
00000100:00000001:3.0:1788575642.006580:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575642.006582:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575642.006583:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575642.006587:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575642.006593:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575642.006594:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575642.006596:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575642.006597:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575642.006599:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575642.006602:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224960:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575642.006604:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575642.006606:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575642.006608:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575642.006620:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575642.006624:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfbc00.
02000000:00000001:3.0:1788575642.006626:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575642.006629:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575642.006631:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5dc0.
00000400:00000010:3.0:1788575642.006634:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575642.006637:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393224960, portal 4
00000100:00000001:3.0:1788575642.006638:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575642.006639:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:3.0:1788575642.006645:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575642.006656:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575642.006658:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575642.006660:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526dd0.
00000100:00000200:3.0:1788575642.006663:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393224960, offset 0
00000400:00000200:3.0:1788575642.006667:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575642.006670:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575642.006673:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575642.006675:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575642.006678:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575642.006679:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575642.006681:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575642.006684:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331013:331013:254:4294967295] 192.168.202.110@tcp LPNI seq info [330970:330970:6:4294967295]
00000400:00000200:3.0:1788575642.006687:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575642.006692:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575642.006696:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575642.006704:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b200.
00000800:00000200:3.0:1788575642.006708:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575642.006711:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575642.006714:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575642.006716:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575642.006719:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575642.006721:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307245753
00000100:00000001:3.0:1788575642.006723:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307246521
00000100:00000001:3.0:1788575642.006725:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307245753
00000100:00000001:3.0:1788575642.006727:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307246521
00000100:00000001:3.0:1788575642.006729:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575642.006731:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575642.006733:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575642.006734:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575642.006737:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575642.006739:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1788575642.006740:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575642.006742:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575642.006743:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575642.006745:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575642.390516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575642.390524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575642.390531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575642.390532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575642.390536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575642.390537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575642.390809:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575642.390814:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575642.390817:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575642.517520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575642.517527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575642.517529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575642.517534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575642.517538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575642.517541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575642.517544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575642.517546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575642.517549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575642.965493:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307246521, at 4307246528
00000100:00000040:1.0:1788575642.965493:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307246521, at 4307246528
00000100:00000010:1.0:1788575642.965508:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:0.0:1788575642.965509:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575643.413510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575643.413515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575643.413519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575643.413520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575643.413522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575643.413523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575643.413576:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575643.413580:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575643.413584:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575643.541478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575643.541485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575643.541487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575643.541492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575643.541495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575643.541499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575643.541501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575643.541504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575643.541507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575644.437476:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575644.437480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575644.437484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575644.437485:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575644.437487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575644.437487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575644.437530:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575644.437534:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575644.437537:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575644.565496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575644.565502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575644.565505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575644.565509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575644.565512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575644.565515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575644.565516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575644.565519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575644.565521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788575645.077689:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788575645.077732:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788575645.077741:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575645.077745:0:2349:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788575645.077749:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575645.077753:0:2349:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788575645.077758:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575645.077762:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bb00.
00000400:00000200:1.0:1788575645.077768:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575645.077773:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575645.077779:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526770
00000400:00000010:1.0:1788575645.077782:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526770.
00000100:00000001:1.0:1788575645.077786:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575645.077801:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575645.077831:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575645.077837:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575645.077844:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575645.077846:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575645.077857:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575645.077859:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728bd00.
00000400:00000200:1.0:1788575645.077862:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575645.077866:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575645.077868:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526b28
00000400:00000010:1.0:1788575645.077870:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526b28.
00000100:00000001:1.0:1788575645.077873:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575645.077877:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575645.077886:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575645.077889:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575645.077895:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575645.077897:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575645.077903:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575645.077906:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b200.
00000400:00000200:1.0:1788575645.077909:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575645.077922:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575645.077925:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526dd0
00000400:00000010:1.0:1788575645.077927:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526dd0.
00000100:00000001:1.0:1788575645.077930:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575645.077933:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575645.077941:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575645.077944:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575667 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575645.077950:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575645.077951:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575645.078008:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575645.078012:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575645.078015:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575645.078023:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575642/real 1788575645]  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575645.078035:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575645.078037:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575645.078040:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c52c0.
00000400:00000200:3.0:1788575645.078044:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526440
00000400:00000010:3.0:1788575645.078045:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526440.
00000100:00000001:3.0:1788575645.078048:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575645.078052:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575645.078062:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078068:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575645.078070:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575645.078074:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575645.078077:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078084:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575645.078088:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078095:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575645.078098:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078099:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575645.078100:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078103:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575645.078106:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575645.078110:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575645.078114:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575645.078117:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575645.078121:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575645.078128:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224832:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575645.078133:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575645.078136:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575642/real 1788575645]  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575645.078144:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575645.078145:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575645.078148:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5dc0.
00000400:00000200:3.0:1788575645.078151:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575645.078152:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575645.078154:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575645.078157:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575645.078164:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078170:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575645.078171:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078173:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575645.078174:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575645.078177:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078183:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575645.078187:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078192:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575645.078195:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078197:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575645.078198:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078205:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575645.078207:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575645.078210:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575645.078213:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575645.078216:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575645.078219:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575645.078296:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393224960:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575645.078301:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575645.078305:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575642/real 1788575645]  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575645.078313:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575645.078315:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575645.078317:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575645.078319:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575645.078320:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575645.078322:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575645.078325:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575645.078332:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078338:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575645.078339:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078342:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575645.078343:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575645.078346:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078354:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575645.078357:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078363:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575645.078365:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078366:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575645.078367:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078369:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575645.078370:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575645.078372:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575645.078375:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575645.078378:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575645.078381:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575645.078388:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393224704:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575645.078423:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078426:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575645.078429:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393224832/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078436:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575645.078440:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575645.078444:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575645.078449:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575645.078453:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575645.078455:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575645.078457:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575645.078460:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575645.078465:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575645.078468:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575645.078470:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:3.0:1788575645.078474:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575645.078475:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078477:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575645.078479:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393224960/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078484:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575645.078485:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575645.078487:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfbc00.
02000000:00000001:3.0:1788575645.078490:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575645.078491:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575645.078492:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575645.078494:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575645.078496:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575645.078498:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575645.078499:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575645.078501:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:3.0:1788575645.078504:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575645.078505:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078507:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575645.078510:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393224704/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575667 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575645.078515:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575645.078516:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575645.078517:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575645.078520:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575645.078521:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575645.078522:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575645.078524:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575645.078525:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575645.078527:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575645.078528:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575645.078529:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575645.078532:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575645.078533:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078537:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575645.078547:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575645.078549:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575645.078551:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575645.078553:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575645.461511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575645.461517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575645.461522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575645.461525:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575645.461527:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575645.461530:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575645.461533:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575645.461536:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575645.461538:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575645.461540:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575645.461543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575645.461545:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575645.461546:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575645.461548:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575645.461550:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575645.461551:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575645.461552:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575645.461555:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575645.461611:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575645.461616:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575645.461644:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575645.589520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575645.589527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575645.589529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575645.589534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575645.589538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575645.589541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575645.589543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575645.589546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575645.589548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575646.485505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575646.485509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575646.485515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575646.485516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575646.485519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575646.485521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575646.485569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575646.485572:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575646.485574:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575646.613511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575646.613518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575646.613521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575646.613526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575646.613531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575646.613534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575646.613537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575646.613539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575646.613542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575647.125503:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575647.125510:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575647.125512:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575647.125515:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575647.125529:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575647.125536:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12578
00000400:00000200:1.0:1788575647.125543:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575647.125549:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575647.125552:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575647.125555:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125558:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575647.125559:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125561:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575647.125564:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575647.125565:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575647.125567:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575647.125569:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575647.125570:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575647.125572:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575647.125573:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125575:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575647.125576:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125577:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575647.125580:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575647.125582:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125584:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575647.125586:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575647.125588:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575647.125590:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575647.125591:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575647.125593:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575647.125597:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788575647.125599:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575647.125602:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575647.125603:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575647.125605:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575647.125607:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575647.125611:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575647.125616:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788575647.125621:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575647.125639:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575647.125650:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575647.125654:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575647.125659:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575647.125685:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575647.125689:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575647.125692:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575647.125693:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575647.125695:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575647.125698:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575647.125701:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12578
00000400:00000200:1.0:1788575647.125705:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575647.125708:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575647.125711:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575647.125713:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125715:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575647.125717:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125719:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575647.125721:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575647.125722:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575647.125724:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575647.125726:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575647.125727:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575647.125729:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575647.125732:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125734:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575647.125736:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125738:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575647.125740:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575647.125743:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125745:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575647.125747:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575647.125752:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575647.125754:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575647.125756:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575647.125760:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575647.125763:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575647.125764:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575647.125768:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788575647.125770:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575647.125773:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575647.125775:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575647.125776:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575647.125778:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575647.125780:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575647.125783:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788575647.125787:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575647.125791:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575647.125799:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575647.125801:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575647.125805:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575647.125812:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575647.125815:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575647.125818:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575647.125819:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575647.125821:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575647.125823:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575647.125826:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12578
00000400:00000200:1.0:1788575647.125829:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575647.125832:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575647.125834:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575647.125837:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125839:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575647.125841:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125843:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575647.125845:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575647.125847:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575647.125848:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575647.125850:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575647.125852:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575647.125855:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575647.125858:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125859:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575647.125861:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125863:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575647.125865:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575647.125867:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575647.125869:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575647.125871:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575647.125874:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575647.125876:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575647.125878:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575647.125879:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575647.125881:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575647.125883:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575647.125886:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788575647.125888:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575647.125891:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575647.125892:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575647.125894:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575647.125895:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575647.125897:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575647.125900:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788575647.125903:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575647.125908:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575647.125915:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575647.125916:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575647.125919:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575647.125925:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575647.125928:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575647.125931:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575647.125936:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12588)
00000100:00000001:3.0:1788575647.125998:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575647.126002:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575647.126004:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575647.126010:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.126018:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575647.126020:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575647.126023:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575647.126026:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575647.126029:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575647.126032:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393225088:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575647.126036:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575647.126041:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575647.126044:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575647.126047:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575647.126051:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575647.126053:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575647.126059:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575647.126063:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575647.126070:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575647.126077:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393225088, portal 25
00000100:00000001:3.0:1788575647.126079:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575647.126081:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345471232 : -130483364080384 : ffff8953796f5500)
00000100:00000040:3.0:1788575647.126086:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.126093:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575647.126096:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575647.126098:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575647.126103:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393225088, offset 0
00000400:00000200:3.0:1788575647.126109:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575647.126118:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575647.126125:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575647.126131:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575647.126135:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575647.126139:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575647.126141:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575647.126145:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331014:331014:256:4294967295] 192.168.202.110@tcp LPNI seq info [330971:330971:8:4294967295]
00000400:00000200:3.0:1788575647.126150:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575647.126161:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575647.126167:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575647.126173:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575647.126179:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575647.126183:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575647.126188:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575647.126191:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575647.126196:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575647.126199:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575647.126204:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575647.126208:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575647.126214:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575647.126217:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575647.126221:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575647.126225:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.126234:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575647.126237:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.126244:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575647.126246:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575647.126253:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575647.126257:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575647.126260:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307251641
00000100:00000010:3.0:1788575647.126266:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575647.126273:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575647.126288:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307251641
00000100:00000010:3.0:1788575647.126292:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575647.126296:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575647.126300:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575647.126303:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575647.126305:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575647.126311:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575647/real 1788575647]  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575647.126379:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575647.126381:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575647.126384:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575647.126387:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575647.126389:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575647.127189:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575647.127198:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575647.127209:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127216:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575647.127218:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127221:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575647.127223:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575647.127227:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127234:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575647.127238:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127245:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575647.127248:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127249:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575647.127250:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127253:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575647.127254:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575647.127258:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575647.127262:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575647.127264:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575647.127269:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575647.127278:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393225088:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575647.127283:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575647.127286:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127291:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575647.127292:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575647.127295:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575647.127297:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575647.127298:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575647.127300:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225216:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575647.127302:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575647.127305:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575647.127307:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575647.127309:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575647.127313:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfbc00.
02000000:00000001:3.0:1788575647.127314:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575647.127319:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575647.127323:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575647.127328:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575647.127332:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393225216, portal 10
00000100:00000001:3.0:1788575647.127334:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575647.127336:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:3.0:1788575647.127342:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127349:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575647.127351:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575647.127353:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575647.127356:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393225216, offset 0
00000400:00000200:3.0:1788575647.127362:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575647.127367:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575647.127374:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575647.127378:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575647.127381:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575647.127383:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575647.127386:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575647.127389:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331015:331015:256:4294967295] 192.168.202.110@tcp LPNI seq info [330972:330972:8:4294967295]
00000400:00000200:3.0:1788575647.127416:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575647.127424:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575647.127429:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575647.127433:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575647.127438:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575647.127442:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575647.127446:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575647.127449:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575647.127453:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575647.127455:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575647.127458:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575647.127461:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575647.127466:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575647.127468:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575647.127471:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575647.127474:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127482:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575647.127485:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127491:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575647.127493:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575647.127498:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575647.127502:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575647.127505:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575647.127509:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307250728
00000100:00000001:3.0:1788575647.127512:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307251641
00000100:00000001:3.0:1788575647.127514:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307250728
00000100:00000001:3.0:1788575647.127515:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307251641
00000100:00000001:3.0:1788575647.127517:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575647.127520:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575647.127521:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575647.127525:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575647/real 1788575647]  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575647.127533:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575647.127535:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575647.127536:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575647.127539:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575647.127541:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575647.127543:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575647.127546:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575647.127554:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127559:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575647.127561:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127563:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575647.127564:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575647.127567:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127572:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575647.127576:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127582:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575647.127585:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127586:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575647.127587:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127589:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575647.127591:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575647.127593:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575647.127596:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575647.127598:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575647.127601:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575647.127608:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225216:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575647.127612:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575647.127615:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127620:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575647.127622:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575647.127635:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575647.127637:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575647.127640:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575647.127642:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575647.127645:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575647.127648:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575647.127649:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575647.127651:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575647.127655:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575647.127656:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575647.127660:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575647.127662:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575647.127665:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575647.127668:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393225344, portal 4
00000100:00000001:3.0:1788575647.127670:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575647.127672:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:3.0:1788575647.127677:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127682:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575647.127684:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575647.127686:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575647.127688:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393225344, offset 0
00000400:00000200:3.0:1788575647.127691:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575647.127695:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575647.127699:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575647.127701:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575647.127704:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575647.127706:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575647.127708:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575647.127710:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331016:331016:256:4294967295] 192.168.202.110@tcp LPNI seq info [330973:330973:8:4294967295]
00000400:00000200:3.0:1788575647.127714:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575647.127720:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575647.127724:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575647.127727:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575647.127731:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575647.127733:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1788575647.127737:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575647.127740:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1788575647.127744:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575647.127746:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575647.127749:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575647.127752:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575647.127755:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575647.127756:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575647.127758:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575647.127762:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127774:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575647.127777:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575672 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127782:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575647.127784:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575647.127787:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575647.127789:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575647.127792:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575647.127794:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307250874
00000100:00000001:3.0:1788575647.127796:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307251641
00000100:00000001:3.0:1788575647.127798:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307250874
00000100:00000001:3.0:1788575647.127800:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307251641
00000100:00000001:3.0:1788575647.127802:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575647.127804:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575647.127806:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575647.127809:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575647/real 1788575647]  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575647.127817:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575647.127818:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575647.127820:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575647.127822:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575647.127823:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575647.127826:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575647.127828:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575647.127835:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127841:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575647.127843:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127844:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575647.127845:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575647.127848:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127855:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575647.127859:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127866:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575647.127868:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127869:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575647.127870:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127871:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575647.127872:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575647.127873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575647.127875:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575647.127876:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575647.127879:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575647.127883:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575647.127886:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127887:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575647.127889:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5500 x1875444393225088/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127893:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575647.127895:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575647.127897:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575647.127899:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575647.127901:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575647.127902:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575647.127904:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575647.127906:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:3.0:1788575647.127908:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575647.127909:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575647.127911:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5500.
00000100:00000001:3.0:1788575647.127913:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575647.127913:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127914:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575647.127916:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393225216/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127920:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575647.127920:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575647.127921:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfbc00.
02000000:00000001:3.0:1788575647.127923:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575647.127923:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575647.127924:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575647.127925:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575647.127925:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:3.0:1788575647.127927:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575647.127927:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575647.127928:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:3.0:1788575647.127929:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575647.127930:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127931:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575647.127932:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393225344/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575672 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575647.127935:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575647.127936:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575647.127937:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575647.127938:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575647.127939:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575647.127939:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575647.127940:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575647.127941:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:3.0:1788575647.127942:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575647.127943:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575647.127944:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:3.0:1788575647.127945:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575647.127946:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127947:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575647.127953:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575647.127955:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575647.127956:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575647.127958:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575647.509502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575647.509509:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575647.509511:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575647.509513:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575647.509516:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575647.509518:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575647.509519:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575647.509521:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575647.509523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575647.509524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575647.509526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575647.509527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575647.509581:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575647.509584:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575647.509586:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575647.637500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575647.637508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575647.637510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575647.637514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575647.637517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575647.637520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575647.637522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575647.637523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575647.637525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575648.085486:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307251641, at 4307251648
00000100:00000040:1.0:1788575648.085486:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307251641, at 4307251648
00000100:00000010:0.0:1788575648.085500:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788575648.085502:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575648.533478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575648.533486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575648.533492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575648.533493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575648.533496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575648.533497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575648.533546:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575648.533550:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575648.533553:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575648.661504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575648.661510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575648.661512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575648.661515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575648.661518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575648.661521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575648.661523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575648.661524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575648.661527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575649.557500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575649.557506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575649.557512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575649.557513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575649.557516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575649.557518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575649.557576:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575649.557580:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575649.557583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575649.685486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575649.685493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575649.685495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575649.685500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575649.685504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575649.685507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575649.685510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575649.685512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575649.685515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575650.581493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575650.581499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575650.581503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575650.581504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575650.581508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575650.581510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575650.581571:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575650.581575:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575650.581578:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575650.709512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575650.709519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575650.709521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575650.709526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575650.709530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575650.709534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575650.709546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575650.709549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575650.709554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575651.605519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575651.605524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575651.605529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575651.605531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575651.605535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575651.605536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575651.605599:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575651.605603:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575651.605607:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575651.733519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575651.733528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575651.733530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575651.733536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575651.733540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575651.733543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575651.733546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575651.733549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575651.733552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575652.245617:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575652.245627:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575652.245629:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575652.245634:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575652.245651:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575652.245658:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12583
00000400:00000200:1.0:1788575652.245667:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575652.245673:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575652.245676:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575652.245688:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245692:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575652.245695:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245697:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575652.245701:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575652.245703:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575652.245706:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575652.245710:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575652.245712:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575652.245715:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575652.245717:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245719:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575652.245721:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245722:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575652.245726:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575652.245736:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245738:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575652.245741:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575652.245744:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575652.245747:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575652.245749:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575652.245752:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575652.245761:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788575652.245764:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575652.245768:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575652.245770:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575652.245772:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575652.245775:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575652.245779:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575652.245786:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788575652.245800:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575652.245810:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575652.245820:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575652.245825:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575652.245830:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575652.245855:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575652.245858:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575652.245861:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575652.245863:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575652.245864:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575652.245867:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575652.245870:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12583
00000400:00000200:1.0:1788575652.245873:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575652.245876:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575652.245879:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575652.245882:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245884:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575652.245886:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245888:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575652.245890:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575652.245891:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575652.245893:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575652.245895:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575652.245897:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575652.245899:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575652.245902:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245904:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575652.245906:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245908:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575652.245910:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575652.245913:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575652.245917:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575652.245919:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575652.245923:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575652.245925:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575652.245927:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575652.245931:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575652.245934:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575652.245936:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575652.245940:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788575652.245942:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575652.245944:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575652.245946:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575652.245947:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575652.245949:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575652.245950:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575652.245954:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788575652.245957:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575652.245962:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575652.245970:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575652.245972:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575652.245976:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575652.245984:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575652.245986:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575652.245990:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575652.245991:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575652.245993:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575652.245995:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575652.245997:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12583
00000400:00000200:1.0:1788575652.246001:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575652.246004:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575652.246006:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575652.246009:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.246011:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575652.246013:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.246015:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575652.246017:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575652.246018:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575652.246020:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575652.246021:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575652.246023:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575652.246025:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575652.246028:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.246030:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575652.246032:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575652.246034:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575652.246036:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575652.246039:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575652.246041:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575652.246042:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575652.246045:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575652.246047:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575652.246049:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575652.246051:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575652.246053:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575652.246055:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575652.246058:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788575652.246060:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575652.246063:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575652.246064:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575652.246065:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575652.246067:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575652.246069:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575652.246073:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788575652.246076:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575652.246080:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575652.246086:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575652.246088:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575652.246092:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575652.246099:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575652.246101:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575652.246105:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575652.246110:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12593)
00000100:00000001:3.0:1788575652.246283:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575652.246294:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575652.246297:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575652.246303:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246311:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575652.246313:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575652.246317:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575652.246320:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575652.246322:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575652.246326:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393225472:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575652.246330:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575652.246334:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575652.246336:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575652.246338:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575652.246343:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575652.246345:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575652.246351:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575652.246355:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575652.246361:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575652.246367:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393225472, portal 25
00000100:00000001:3.0:1788575652.246370:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575652.246371:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00000040:3.0:1788575652.246377:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246384:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575652.246387:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575652.246413:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575652.246421:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393225472, offset 0
00000400:00000200:3.0:1788575652.246429:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575652.246437:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575652.246443:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575652.246448:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575652.246452:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575652.246455:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575652.246458:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575652.246461:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331017:331017:256:4294967295] 192.168.202.110@tcp LPNI seq info [330974:330974:8:4294967295]
00000400:00000200:3.0:1788575652.246465:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575652.246474:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575652.246478:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575652.246483:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575652.246489:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575652.246492:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575652.246497:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575652.246500:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575652.246504:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575652.246506:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575652.246512:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575652.246516:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575652.246521:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575652.246523:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575652.246527:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575652.246531:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246539:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575652.246542:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575677 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246549:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575652.246550:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575652.246556:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575652.246560:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575652.246563:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307256762
00000100:00000010:3.0:1788575652.246568:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575652.246574:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575652.246588:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307256762
00000100:00000010:3.0:1788575652.246592:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575652.246596:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575652.246599:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575652.246601:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575652.246603:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575652.246609:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575652/real 1788575652]  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575652.246618:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575652.246620:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575652.246623:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575652.246625:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575652.246627:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575652.246630:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575652.246633:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575652.246654:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246661:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575652.246663:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.246665:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575652.246667:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575652.246671:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246678:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575652.246682:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246690:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575652.246693:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.246695:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575652.246696:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.246698:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575652.246700:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575652.246703:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575652.246706:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575652.246709:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575652.246713:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575652.246721:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393225472:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575652.246725:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575652.246728:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246735:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575652.246736:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575652.246738:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575652.246739:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575652.246741:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575652.246744:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575652.246747:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575652.246748:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575652.246750:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575652.246751:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575652.246755:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfbc00.
02000000:00000001:3.0:1788575652.246756:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575652.246760:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575652.246763:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575652.246765:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575652.246768:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393225600, portal 10
00000100:00000001:3.0:1788575652.246770:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575652.246771:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:3.0:1788575652.246776:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246782:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575652.246784:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575652.246786:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575652.246788:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393225600, offset 0
00000400:00000200:3.0:1788575652.246792:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575652.246795:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575652.246799:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575652.246802:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575652.246804:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575652.246807:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575652.246809:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575652.246812:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331018:331018:256:4294967295] 192.168.202.110@tcp LPNI seq info [330975:330975:8:4294967295]
00000400:00000200:3.0:1788575652.246816:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575652.246822:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575652.246826:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575652.246829:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575652.246833:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575652.246836:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575652.246840:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575652.246842:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575652.246846:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575652.246848:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575652.246850:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575652.246854:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575652.246856:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575652.246858:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575652.246861:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575652.246864:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246872:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575652.246875:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575677 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246882:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575652.246883:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575652.246886:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575652.246889:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575652.246892:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575652.246894:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307255847
00000100:00000001:3.0:1788575652.246897:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307256762
00000100:00000001:3.0:1788575652.246899:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307255847
00000100:00000001:3.0:1788575652.246901:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307256762
00000100:00000001:3.0:1788575652.246902:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575652.246904:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575652.246906:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575652.246910:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575652/real 1788575652]  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575652.246917:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575652.246918:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575652.246920:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575652.246922:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575652.246924:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575652.246926:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575652.246929:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575652.246937:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246948:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575652.246950:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.246951:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575652.246953:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575652.246956:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246963:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575652.246967:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.246973:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575652.246975:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.246977:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575652.246978:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.246980:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575652.246982:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575652.246984:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575652.246987:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575652.246990:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575652.246994:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575652.247001:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575652.247005:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575652.247008:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247014:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575652.247016:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575652.247018:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575652.247020:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575652.247022:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575652.247025:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225728:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575652.247027:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575652.247029:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575652.247031:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575652.247032:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575652.247036:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575652.247038:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575652.247042:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575652.247044:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575652.247048:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575652.247050:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393225728, portal 4
00000100:00000001:3.0:1788575652.247052:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575652.247054:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000040:3.0:1788575652.247059:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247065:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575652.247067:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575652.247070:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575652.247072:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393225728, offset 0
00000400:00000200:3.0:1788575652.247076:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575652.247079:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575652.247084:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575652.247086:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575652.247088:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575652.247090:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575652.247092:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575652.247095:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331019:331019:256:4294967295] 192.168.202.110@tcp LPNI seq info [330976:330976:8:4294967295]
00000400:00000200:3.0:1788575652.247099:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575652.247105:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575652.247110:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575652.247114:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575652.247117:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575652.247120:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1788575652.247124:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575652.247127:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1788575652.247130:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575652.247132:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575652.247135:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575652.247138:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575652.247141:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575652.247142:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575652.247145:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575652.247148:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575677 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247155:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575652.247158:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575677 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247164:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575652.247166:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575652.247169:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575652.247172:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575652.247176:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575652.247178:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307255993
00000100:00000001:3.0:1788575652.247180:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307256762
00000100:00000001:3.0:1788575652.247182:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307255993
00000100:00000001:3.0:1788575652.247184:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307256762
00000100:00000001:3.0:1788575652.247186:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575652.247188:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575652.247190:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575652.247193:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575652/real 1788575652]  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575652.247201:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575652.247203:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575652.247204:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575652.247206:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575652.247208:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575652.247210:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575652.247213:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575652.247221:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247227:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575652.247228:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.247230:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575652.247231:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575652.247235:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247241:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575652.247245:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247251:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575652.247254:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.247255:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575652.247257:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.247259:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575652.247260:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575652.247262:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575652.247266:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575652.247268:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575652.247272:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575652.247280:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225728:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575652.247284:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.247286:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575652.247289:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393225472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247295:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575652.247298:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575652.247300:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfa800.
02000000:00000001:3.0:1788575652.247303:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575652.247305:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575652.247306:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575652.247309:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575652.247311:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:3.0:1788575652.247315:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575652.247316:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575652.247318:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:3.0:1788575652.247321:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575652.247322:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.247324:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575652.247327:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393225600/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247333:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575652.247334:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575652.247336:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfbc00.
02000000:00000001:3.0:1788575652.247338:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575652.247339:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575652.247340:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575652.247342:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575652.247343:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:3.0:1788575652.247345:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575652.247346:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575652.247348:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:3.0:1788575652.247350:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575652.247351:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.247353:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575652.247355:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393225728/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575677 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575652.247361:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575652.247362:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575652.247364:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575652.247365:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575652.247367:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575652.247368:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575652.247370:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575652.247371:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:3.0:1788575652.247373:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575652.247374:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575652.247376:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:3.0:1788575652.247378:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575652.247379:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.247381:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575652.247387:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575652.247389:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575652.247408:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575652.247411:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575652.629529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575652.629534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575652.629538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575652.629539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575652.629541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575652.629542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575652.629587:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575652.629589:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575652.629591:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575652.757511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575652.757516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575652.757518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575652.757522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575652.757524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575652.757527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575652.757528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575652.757531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575652.757533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575653.205500:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307256762, at 4307256768
00000100:00000040:1.0:1788575653.205500:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307256762, at 4307256768
00000100:00000010:1.0:1788575653.205514:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575653.205515:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575653.653518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575653.653525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575653.653529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575653.653529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575653.653532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575653.653532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575653.653592:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575653.653595:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575653.653599:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575653.781502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575653.781509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575653.781511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575653.781515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575653.781518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575653.781521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575653.781525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575653.781528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575653.781531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575654.677465:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575654.677479:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575654.677485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575654.677486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575654.677489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575654.677491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575654.677537:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575654.677540:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575654.677542:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575654.805499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575654.805505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575654.805507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575654.805512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575654.805515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575654.805518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575654.805521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575654.805523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575654.805526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575655.701504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575655.701510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575655.701516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575655.701519:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575655.701521:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575655.701524:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575655.701527:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575655.701528:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575655.701531:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575655.701533:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575655.701536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575655.701537:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575655.701538:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575655.701539:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575655.701541:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575655.701542:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575655.701544:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575655.701545:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575655.701601:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575655.701605:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575655.701608:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575655.829478:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575655.829485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575655.829487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575655.829492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575655.829495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575655.829500:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575655.829503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575655.829506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575655.829508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575656.725525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575656.725531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575656.725537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575656.725539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575656.725542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575656.725544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575656.725612:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575656.725616:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575656.725619:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575656.853554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575656.853561:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575656.853564:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575656.853569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575656.853572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575656.853577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575656.853581:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575656.853583:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575656.853586:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575657.365523:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575657.365534:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575657.365536:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575657.365540:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575657.365544:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575657.365551:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12588
00000400:00000200:3.0:1788575657.365560:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575657.365567:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575657.365570:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575657.365573:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365576:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575657.365579:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365581:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575657.365585:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575657.365587:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575657.365591:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575657.365594:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575657.365596:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575657.365598:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575657.365601:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365603:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575657.365605:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365606:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575657.365610:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575657.365613:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365615:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575657.365618:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575657.365621:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575657.365623:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575657.365626:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575657.365629:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575657.365649:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575657.365653:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575657.365657:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575657.365658:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575657.365661:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575657.365663:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575657.365667:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575657.365675:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:3.0:1788575657.365681:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575657.365689:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575657.365699:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575657.365704:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575657.365708:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575657.365731:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575657.365734:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575657.365737:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575657.365738:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575657.365740:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575657.365743:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575657.365745:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12588
00000400:00000200:3.0:1788575657.365749:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575657.365752:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575657.365754:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575657.365758:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365760:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575657.365763:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365765:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575657.365768:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575657.365769:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575657.365770:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575657.365771:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1788575657.365773:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575657.365775:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575657.365777:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575657.365779:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575657.365781:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575657.365786:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575657.365787:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365792:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575657.365797:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365798:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575657.365800:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575657.365806:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575657.365808:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575657.365812:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575657.365812:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575657.365818:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575657.365820:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575657.365822:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575657.365824:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575657.365826:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1788575657.365828:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393225856:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575657.365830:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.365832:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575657.365835:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575657.365837:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575657.365840:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575657.365841:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575657.365842:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:3.0:1788575657.365846:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575657.365848:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
00000100:00000001:3.0:1788575657.365849:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575657.365851:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575657.365856:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000400:00000200:2.0:1788575657.365857:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1788575657.365862:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:2.0:1788575657.365863:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
02000000:00000001:3.0:1788575657.365868:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1788575657.365871:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
02000000:00000001:3.0:1788575657.365872:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575657.365876:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:2.0:1788575657.365877:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393225856, portal 25
02000000:00000001:3.0:1788575657.365878:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575657.365881:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1788575657.365882:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.365883:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:2.0:1788575657.365891:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788575657.365892:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8c00.
00000100:00000001:3.0:1788575657.365898:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.365899:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575657.365903:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575657.365905:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00100000:3.0:1788575657.365907:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788575657.365908:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393225856, offset 0
00000400:00000200:2.0:1788575657.365919:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575657.365924:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575657.365927:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575657.365928:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575657.365935:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575657.365936:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575657.365943:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575657.365949:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575657.365952:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575657.365956:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1788575657.365957:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575657.365962:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:3.0:1788575657.365964:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575657.365968:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575657.365971:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575657.365972:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331020:331020:256:4294967295] 192.168.202.110@tcp LPNI seq info [330977:330977:8:4294967295]
00000100:00000001:3.0:1788575657.365975:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575657.365980:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12588
00000400:00000200:2.0:1788575657.365983:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575657.365990:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575657.365998:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1788575657.365998:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575657.366003:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1788575657.366007:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575657.366009:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575657.366012:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000100:00000001:3.0:1788575657.366013:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575657.366017:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.0:1788575657.366021:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575657.366023:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.0:1788575657.366025:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:3.0:1788575657.366027:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575657.366031:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:2.1:1788575657.366032:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788575657.366035:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.1:1788575657.366036:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000040:3.0:1788575657.366040:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575657.366043:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575657.366045:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575657.366052:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000100:2.0:1788575657.366055:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575657.366056:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000010:2.0:1788575657.366059:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000100:00000040:3.0:1788575657.366062:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575657.366065:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575657.366068:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:3.0:1788575657.366068:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575657.366075:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575657.366080:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1788575657.366081:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575657.366086:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575657.366088:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
02000000:00000001:3.0:1788575657.366089:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:2.0:1788575657.366094:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
02000000:00000001:3.0:1788575657.366094:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575657.366098:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575657.366102:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000008:00100000:3.0:1788575657.366102:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575657.366107:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575657.366111:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575657.366111:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575657.366117:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788575657.366122:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575657.366126:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575657.366129:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.366130:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1788575657.366132:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575657.366135:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:2.0:1788575657.366137:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575682 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575657.366137:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575657.366144:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000100:00000001:2.0:1788575657.366150:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575657.366151:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.366152:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575657.366161:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575657.366161:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575657.366169:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575657.366173:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307261881
00000100:00000001:3.0:1788575657.366177:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000010:2.0:1788575657.366179:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c1e0.
00000100:00000001:3.0:1788575657.366181:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.366187:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:3.0:1788575657.366188:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575657.366197:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575657.366200:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575657.366202:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307261881
00000100:00000010:2.0:1788575657.366208:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c7e0.
00000100:00000040:3.0:1788575657.366208:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575657.366213:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1788575657.366216:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12598)
00000100:00000001:2.0:1788575657.366225:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575657.366230:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.366232:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575657.366239:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575657/real 1788575657]  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575657.366249:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575657.366251:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575657.366254:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575657.366258:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575657.366260:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575657.366263:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575657.366266:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575657.366279:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366285:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575657.366287:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366289:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575657.366291:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575657.366295:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366302:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575657.366306:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366313:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575657.366316:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366318:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575657.366319:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366321:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575657.366323:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575657.366325:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575657.366329:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575657.366330:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575657.366333:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575657.366338:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393225856:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575657.366341:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366343:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575657.366346:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393225856/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366358:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575657.366360:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575657.366363:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575657.366366:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575657.366367:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575657.366369:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575657.366371:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575657.366373:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575657.366377:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575657.366389:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575657.366414:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575657.366418:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575657.366419:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366422:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366428:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575657.366430:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.366432:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575657.366434:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575657.366435:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575657.366439:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366445:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575657.366447:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575657.366449:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575657.366450:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575657.366452:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575657.366454:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225984:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575657.366456:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575657.366458:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575657.366459:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575657.366460:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575657.366463:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575657.366464:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575657.366468:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575657.366471:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575657.366475:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575657.366478:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393225984, portal 10
00000100:00000001:2.0:1788575657.366479:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575657.366481:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:2.0:1788575657.366487:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366492:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575657.366495:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575657.366497:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575657.366499:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393225984, offset 0
00000400:00000200:2.0:1788575657.366503:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575657.366507:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575657.366510:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575657.366513:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575657.366515:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575657.366518:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575657.366520:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575657.366523:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331021:331021:256:4294967295] 192.168.202.110@tcp LPNI seq info [330978:330978:8:4294967295]
00000400:00000200:2.0:1788575657.366527:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575657.366533:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575657.366538:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575657.366541:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575657.366545:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575657.366547:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575657.366551:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575657.366554:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1788575657.366557:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575657.366559:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575657.366561:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575657.366565:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575657.366568:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575657.366570:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575657.366573:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575657.366576:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366584:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575657.366587:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575682 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366592:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.366594:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575657.366597:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575657.366601:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575657.366604:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575657.366606:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307260967
00000100:00000001:2.0:1788575657.366609:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307261881
00000100:00000001:2.0:1788575657.366611:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307260967
00000100:00000001:2.0:1788575657.366613:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307261881
00000100:00000001:2.0:1788575657.366615:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575657.366617:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.366619:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575657.366623:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575657/real 1788575657]  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575657.366631:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575657.366633:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575657.366634:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575657.366650:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575657.366651:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575657.366654:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575657.366657:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575657.366665:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366688:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575657.366689:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366691:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575657.366692:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575657.366696:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366703:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575657.366707:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366713:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575657.366715:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366717:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575657.366718:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366720:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575657.366722:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575657.366724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575657.366727:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575657.366730:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575657.366733:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575657.366741:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393225984:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575657.366744:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575657.366747:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366752:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575657.366753:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575657.366755:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575657.366756:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575657.366757:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575657.366759:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226112:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575657.366761:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575657.366762:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575657.366763:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575657.366764:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575657.366767:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575657.366768:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575657.366771:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575657.366773:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575657.366775:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575657.366777:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393226112, portal 4
00000100:00000001:2.0:1788575657.366778:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575657.366779:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788575657.366783:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366786:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575657.366788:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575657.366789:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575657.366791:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393226112, offset 0
00000400:00000200:2.0:1788575657.366793:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575657.366795:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575657.366798:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575657.366799:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575657.366801:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575657.366802:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575657.366804:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575657.366806:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331022:331022:256:4294967295] 192.168.202.110@tcp LPNI seq info [330979:330979:8:4294967295]
00000400:00000200:2.0:1788575657.366808:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575657.366812:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575657.366815:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575657.366817:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575657.366820:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575657.366821:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575657.366824:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575657.366825:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1788575657.366828:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575657.366829:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575657.366831:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575657.366835:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575657.366837:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575657.366839:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575657.366841:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575657.366844:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575682 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366851:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575657.366854:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575682 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366859:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.366861:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575657.366864:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575657.366866:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575657.366870:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575657.366872:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307261113
00000100:00000001:2.0:1788575657.366874:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307261881
00000100:00000001:2.0:1788575657.366876:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307261113
00000100:00000001:2.0:1788575657.366878:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307261881
00000100:00000001:2.0:1788575657.366879:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575657.366881:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.366882:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575657.366885:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575657/real 1788575657]  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575657.366890:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575657.366891:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575657.366892:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575657.366893:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575657.366894:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575657.366896:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575657.366897:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575657.366902:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366906:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575657.366906:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366907:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575657.366908:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575657.366910:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366914:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575657.366917:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366923:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575657.366925:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366927:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575657.366928:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366930:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575657.366931:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575657.366933:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575657.366936:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575657.366938:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575657.366943:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575657.366950:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226112:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575657.366954:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366956:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575657.366959:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393225984/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366966:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575657.366967:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575657.366968:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575657.366970:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575657.366971:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575657.366972:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575657.366973:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575657.366974:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:2.0:1788575657.366976:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575657.366977:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575657.366977:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:2.0:1788575657.366979:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575657.366980:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.366981:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575657.366983:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393226112/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575682 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575657.366986:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575657.366987:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575657.366988:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575657.366989:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575657.366990:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575657.366991:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575657.366992:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575657.366992:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:2.0:1788575657.366993:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575657.366995:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575657.366996:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575657.366998:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575657.366999:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.367001:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.367004:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575657.367005:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575657.367007:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575657.367008:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575657.749548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575657.749561:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575657.749564:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575657.749566:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575657.749572:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575657.749573:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575657.749575:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575657.749577:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575657.749580:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575657.749581:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575657.749584:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575657.749584:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575657.749635:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575657.749652:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575657.749657:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575657.877500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575657.877507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575657.877509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575657.877514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575657.877517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575657.877520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.877523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575657.877525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575657.877528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575658.325510:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307261881, at 4307261888
00000100:00000040:1.0:1788575658.325510:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307261881, at 4307261888
00000100:00000010:1.0:1788575658.325526:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000100:00000010:0.0:1788575658.325530:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00010000:00000001:3.0:1788575658.773514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575658.773520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575658.773525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575658.773526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575658.773529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575658.773530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575658.773585:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575658.773589:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575658.773591:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575658.901536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575658.901548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575658.901552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575658.901571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575658.901576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575658.901580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575658.901583:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575658.901586:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575658.901588:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575659.797506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575659.797512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575659.797518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575659.797519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575659.797523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575659.797524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575659.797572:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575659.797576:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575659.797579:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575659.925497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575659.925502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575659.925504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575659.925507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575659.925511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575659.925514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575659.925516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575659.925518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575659.925520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575660.821729:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575660.821735:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575660.821739:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575660.821740:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575660.821743:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575660.821744:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575660.821897:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575660.821900:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575660.821902:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575660.949520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575660.949527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575660.949529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575660.949533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575660.949536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575660.949539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575660.949541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575660.949543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575660.949545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575661.845542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575661.845549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575661.845555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575661.845556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575661.845559:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575661.845560:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575661.845628:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575661.845632:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575661.845647:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575661.973506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575661.973513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575661.973515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575661.973520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575661.973524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575661.973531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575661.973533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575661.973536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575661.973538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575662.485516:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575662.485526:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575662.485529:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575662.485533:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575662.485537:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575662.485545:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12593
00000400:00000200:2.0:1788575662.485553:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575662.485559:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575662.485562:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575662.485565:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485568:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575662.485570:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485573:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575662.485578:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575662.485579:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575662.485582:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575662.485584:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575662.485586:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575662.485587:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575662.485589:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485590:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575662.485591:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485592:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575662.485594:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575662.485597:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485599:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575662.485602:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575662.485605:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575662.485608:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575662.485610:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575662.485612:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575662.485621:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02680.
00000020:00000040:2.0:1788575662.485630:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575662.485634:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575662.485662:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575662.485665:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575662.485667:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575662.485670:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575662.485678:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:2.0:1788575662.485685:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575662.485693:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575662.485703:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575662.485707:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575662.485712:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000450d1599] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575662.485733:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575662.485735:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575662.485738:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575662.485739:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575662.485740:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575662.485742:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575662.485744:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12593
00000400:00000200:2.0:1788575662.485747:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575662.485750:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575662.485752:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575662.485754:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485756:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575662.485758:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485759:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575662.485760:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575662.485761:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575662.485762:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575662.485763:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575662.485766:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575662.485768:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575662.485769:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575662.485770:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485771:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575662.485773:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485774:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575662.485775:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1788575662.485776:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575662.485779:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575662.485779:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788575662.485783:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575662.485784:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575662.485789:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1788575662.485789:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575662.485793:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575662.485796:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575662.485800:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575662.485804:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575662.485808:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575662.485808:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000010:2.0:1788575662.485811:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575662.485812:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575662.485813:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575662.485817:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575662.485818:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575662.485819:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575662.485819:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575662.485822:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575662.485824:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575662.485824:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:2.0:1788575662.485828:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575662.485830:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575662.485832:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575662.485837:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393226240:192.168.202.110@tcp:250:kworker.0
00000100:00100000:2.0:1788575662.485838:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575662.485847:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1788575662.485850:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1788575662.485853:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1788575662.485854:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575662.485857:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575662.485861:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e69941e] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575662.485861:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575662.485867:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575662.485870:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575662.485874:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575662.485878:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575662.485880:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1788575662.485882:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00080000:2.0:1788575662.485887:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:3.0:1788575662.485889:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575662.485890:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575662.485893:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1788575662.485896:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393226240, portal 25
00000100:00000001:2.0:1788575662.485897:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:3.0:1788575662.485899:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575662.485901:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00080000:2.0:1788575662.485902:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12593
00000400:00000200:2.0:1788575662.485909:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575662.485909:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575662.485913:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575662.485918:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575662.485920:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575662.485925:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575662.485925:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575662.485928:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000001:2.0:1788575662.485929:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1788575662.485931:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393226240, offset 0
00000100:00000040:2.0:1788575662.485935:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575662.485939:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485941:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575662.485945:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575662.485946:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575662.485948:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575662.485950:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575662.485952:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575662.485955:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575662.485956:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575662.485961:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485963:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575662.485966:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1788575662.485967:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485971:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575662.485973:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575662.485976:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575662.485976:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788575662.485980:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575662.485982:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575662.485984:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:2.0:1788575662.485985:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575662.485988:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575662.485990:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575662.485990:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00100000:2.0:1788575662.485992:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575662.485995:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575662.485997:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575662.485998:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575662.486003:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000020:00000040:2.0:1788575662.486006:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575662.486007:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331023:331023:256:4294967295] 192.168.202.110@tcp LPNI seq info [330980:330980:8:4294967295]
02000000:00000001:2.0:1788575662.486009:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575662.486011:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575662.486015:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575662.486016:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575662.486018:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575662.486019:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1788575662.486022:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788575662.486026:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575662.486031:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575662.486040:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1788575662.486042:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575662.486045:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575662.486049:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788575662.486052:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575662.486061:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000100:00000001:2.0:1788575662.486070:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575662.486070:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575662.486073:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.0:1788575662.486076:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:2.0:1788575662.486077:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:3.1:1788575662.486085:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575662.486087:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12603)
00000800:00000200:3.1:1788575662.486089:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575662.486095:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575662.486098:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575662.486105:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575662.486110:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575662.486115:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575662.486118:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575662.486122:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575662.486126:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486135:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575662.486139:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575687 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486145:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575662.486147:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575662.486154:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575662.486158:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575662.486161:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307267001
00000100:00000010:3.0:1788575662.486166:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575662.486172:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575662.486182:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307267001
00000100:00000010:3.0:1788575662.486185:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575662.486187:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575662.486189:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575662.486191:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575662.486192:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575662.486196:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575662/real 1788575662]  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575662.486203:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575662.486204:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575662.486206:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575662.486208:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575662.486209:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575662.486211:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575662.486213:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575662.486218:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486223:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575662.486224:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486226:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575662.486227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575662.486229:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486233:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575662.486236:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486240:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575662.486242:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486243:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575662.486244:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486245:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575662.486246:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575662.486248:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575662.486250:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575662.486252:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575662.486254:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575662.486258:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393226240:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575662.486261:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486262:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575662.486264:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393226240/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486272:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575662.486273:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575662.486275:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575662.486277:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575662.486278:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575662.486279:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575662.486281:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575662.486283:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575662.486285:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575662.486286:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575662.486287:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575662.486289:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575662.486290:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486291:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486296:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575662.486301:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575662.486302:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575662.486303:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575662.486304:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575662.486306:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486311:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575662.486311:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575662.486313:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575662.486314:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575662.486315:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575662.486317:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575662.486319:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575662.486320:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575662.486321:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575662.486322:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575662.486324:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575662.486325:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575662.486327:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575662.486329:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575662.486331:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575662.486333:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393226368, portal 10
00000100:00000001:3.0:1788575662.486334:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575662.486335:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:3.0:1788575662.486339:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486343:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575662.486344:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575662.486345:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575662.486347:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393226368, offset 0
00000400:00000200:3.0:1788575662.486349:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575662.486351:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575662.486353:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575662.486355:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575662.486357:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575662.486358:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575662.486360:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575662.486362:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331024:331024:256:4294967295] 192.168.202.110@tcp LPNI seq info [330981:330981:8:4294967295]
00000400:00000200:3.0:1788575662.486364:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575662.486368:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575662.486371:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575662.486373:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575662.486375:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575662.486377:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575662.486380:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575662.486381:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575662.486384:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575662.486385:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575662.486387:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575662.486389:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575662.486410:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575662.486412:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575662.486415:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575662.486418:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486425:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575662.486428:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575687 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486435:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575662.486436:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575662.486440:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575662.486444:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575662.486447:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575662.486449:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307266087
00000100:00000001:3.0:1788575662.486451:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307267001
00000100:00000001:3.0:1788575662.486453:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307266087
00000100:00000001:3.0:1788575662.486455:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307267001
00000100:00000001:3.0:1788575662.486457:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575662.486459:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575662.486461:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575662.486465:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575662/real 1788575662]  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575662.486472:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575662.486474:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575662.486475:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575662.486478:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575662.486480:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575662.486482:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575662.486485:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575662.486492:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486503:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575662.486504:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486506:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575662.486507:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575662.486511:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486515:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575662.486518:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486521:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575662.486523:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575662.486524:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486525:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575662.486526:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575662.486528:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575662.486530:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575662.486531:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575662.486533:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575662.486539:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575662.486543:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575662.486546:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486552:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575662.486553:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575662.486556:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575662.486558:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575662.486560:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575662.486563:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575662.486565:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575662.486567:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575662.486569:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575662.486578:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575662.486582:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575662.486584:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575662.486587:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575662.486590:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575662.486593:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575662.486596:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393226496, portal 4
00000100:00000001:3.0:1788575662.486598:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575662.486599:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575662.486605:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486611:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575662.486614:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575662.486616:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575662.486619:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393226496, offset 0
00000400:00000200:3.0:1788575662.486622:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575662.486626:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575662.486629:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575662.486632:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575662.486645:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575662.486647:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575662.486648:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575662.486650:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331025:331025:256:4294967295] 192.168.202.110@tcp LPNI seq info [330982:330982:8:4294967295]
00000400:00000200:3.0:1788575662.486653:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575662.486657:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575662.486660:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575662.486663:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575662.486665:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575662.486667:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575662.486671:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575662.486673:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575662.486677:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575662.486679:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575662.486686:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575662.486690:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575662.486693:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575662.486694:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575662.486696:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575662.486700:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575687 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486707:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575662.486710:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575687 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486716:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575662.486717:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575662.486725:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575662.486728:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575662.486731:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575662.486733:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307266233
00000100:00000001:3.0:1788575662.486735:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307267001
00000100:00000001:3.0:1788575662.486737:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307266233
00000100:00000001:3.0:1788575662.486739:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307267001
00000100:00000001:3.0:1788575662.486741:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575662.486743:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575662.486745:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575662.486748:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575662/real 1788575662]  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575662.486755:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575662.486756:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575662.486757:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575662.486759:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575662.486760:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575662.486761:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575662.486763:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575662.486768:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486771:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575662.486772:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486773:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575662.486774:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575662.486776:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486779:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575662.486781:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486785:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575662.486787:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486788:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575662.486789:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486790:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575662.486791:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575662.486792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575662.486794:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575662.486795:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575662.486797:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575662.486802:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575662.486804:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486805:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575662.486807:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393226368/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486810:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575662.486811:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575662.486812:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575662.486814:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575662.486815:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575662.486816:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575662.486817:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575662.486818:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575662.486820:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575662.486821:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575662.486822:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575662.486823:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575662.486824:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486825:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575662.486826:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393226496/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575687 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575662.486830:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575662.486831:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575662.486831:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575662.486832:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575662.486833:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575662.486834:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575662.486835:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575662.486835:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575662.486836:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575662.486837:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575662.486838:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575662.486839:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575662.486840:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486841:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575662.486844:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575662.486845:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575662.486846:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575662.486847:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575662.869619:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575662.869625:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575662.869628:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575662.869647:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575662.869660:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575662.869661:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575662.869711:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575662.869714:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575662.869716:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575662.997460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575662.997470:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575662.997472:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575662.997477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575662.997480:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575662.997483:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575662.997485:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575662.997488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575662.997490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575663.445468:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307267001, at 4307267008
00000100:00000040:1.0:1788575663.445468:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307267001, at 4307267008
00000100:00000010:0.0:1788575663.445482:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:1.0:1788575663.445484:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575663.893494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575663.893499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575663.893502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575663.893503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575663.893505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575663.893506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575663.893558:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575663.893561:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575663.893564:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575664.021510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575664.021518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575664.021521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575664.021526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575664.021530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575664.021534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575664.021537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575664.021540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575664.021542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575664.917561:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575664.917568:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575664.917573:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575664.917586:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575664.917589:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575664.917592:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575664.917595:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575664.917597:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575664.917599:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575664.917602:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575664.917606:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575664.917607:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575664.917608:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575664.917610:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575664.917612:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575664.917613:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575664.917615:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575664.917617:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575664.917682:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575664.917685:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575664.917689:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575665.045518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575665.045526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575665.045528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575665.045534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575665.045538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575665.045542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575665.045544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575665.045547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575665.045549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575665.941764:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575665.941770:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575665.941776:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575665.941777:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575665.941781:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575665.941782:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575665.941876:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575665.941880:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575665.941884:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575666.069487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575666.069494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575666.069496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575666.069503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575666.069507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575666.069511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575666.069514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575666.069517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575666.069521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575666.965520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575666.965526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575666.965542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575666.965544:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575666.965547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575666.965548:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575666.965599:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575666.965603:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575666.965605:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575667.093469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575667.093474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575667.093476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575667.093479:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575667.093481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575667.093484:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575667.093486:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575667.093488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575667.093489:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575667.605581:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575667.605590:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575667.605593:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575667.605596:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575667.605600:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575667.605607:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12598
00000400:00000200:1.0:1788575667.605615:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575667.605621:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575667.605624:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575667.605628:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575667.605630:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575667.605632:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575667.605651:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575667.605655:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575667.605657:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575667.605659:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575667.605662:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575667.605665:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575667.605667:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575667.605670:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575667.605672:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575667.605674:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575667.605676:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575667.605679:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575667.605683:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575667.605685:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575667.605688:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575667.605691:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575667.605694:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575667.605696:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575667.605698:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575667.605708:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788575667.605712:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575667.605716:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575667.605718:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575667.605721:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575667.605723:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575667.605728:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575667.605736:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788575667.605742:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575667.605751:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575667.605761:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575667.605773:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575667.605779:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575667.605797:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575667.605800:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575667.605804:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575667.605805:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575667.605807:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575667.605809:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575667.605812:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12598
00000400:00000200:1.0:1788575667.605816:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575667.605820:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575667.605822:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575667.605825:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575667.605827:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575667.605830:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575667.605832:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575667.605835:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1788575667.605835:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:1.0:1788575667.605838:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575667.605841:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575667.605841:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575667.605845:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1788575667.605846:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575667.605850:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575667.605853:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575667.605854:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575667.605859:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575667.605864:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575667.605869:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575667.605871:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575667.605874:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575667.605875:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788575667.605880:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575667.605882:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575667.605886:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1788575667.605888:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575667.605890:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575667.605894:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575667.605898:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393226624:192.168.202.110@tcp:250:kworker.0
02000000:00000001:1.0:1788575667.605899:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575667.605906:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788575667.605907:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575667.605911:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575667.605911:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575667.605915:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575667.605916:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575667.605920:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:1.0:1788575667.605922:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:3.0:1788575667.605928:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575667.605931:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1788575667.605935:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575667.605936:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575667.605940:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575667.605942:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000010:1.0:1788575667.605947:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788575667.605951:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:3.0:1788575667.605952:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000001:1.0:1788575667.605954:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575667.605957:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575667.605959:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575667.605960:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:3.0:1788575667.605960:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393226624, portal 25
02000000:00000001:1.0:1788575667.605963:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.605965:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575667.605968:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
02000000:00000010:1.0:1788575667.605970:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788575667.605975:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575667.605976:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788575667.605985:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575667.605988:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575667.605997:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575667.605998:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575667.606001:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575667.606001:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575667.606008:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393226624, offset 0
00000100:00000040:1.0:1788575667.606009:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575667.606021:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575667.606022:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575667.606026:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575667.606029:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:3.0:1788575667.606031:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575667.606032:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575667.606036:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575667.606039:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:3.0:1788575667.606042:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:1.0:1788575667.606045:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12598
00000400:00000200:3.0:1788575667.606050:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788575667.606052:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575667.606056:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788575667.606060:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1788575667.606061:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575667.606065:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:1.0:1788575667.606066:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575667.606071:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331026:331026:256:4294967295] 192.168.202.110@tcp LPNI seq info [330983:330983:8:4294967295]
00000100:00000040:1.0:1788575667.606074:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575667.606078:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575667.606081:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575667.606085:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575667.606091:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575667.606091:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000001:1.0:1788575667.606097:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:3.0:1788575667.606100:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:1.0:1788575667.606101:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575667.606105:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000010:3.0:1788575667.606106:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000020:00000040:1.0:1788575667.606107:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575667.606112:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575667.606116:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.0:1788575667.606117:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788575667.606123:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575667.606124:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:1.0:1788575667.606128:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575667.606133:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575667.606137:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788575667.606138:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:3.1:1788575667.606142:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000040:1.0:1788575667.606143:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575667.606151:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000100:3.0:1788575667.606153:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575667.606157:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575667.606158:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
02000000:00000001:1.0:1788575667.606161:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575667.606166:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575667.606167:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:1.0:1788575667.606171:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575667.606176:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575667.606176:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000008:00100000:1.0:1788575667.606180:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:3.0:1788575667.606184:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000008:00000001:1.0:1788575667.606185:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575667.606188:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:1.0:1788575667.606189:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.606195:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000010:1.0:1788575667.606196:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000100:00000200:3.0:1788575667.606202:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788575667.606203:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575667.606208:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575667.606211:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575667.606215:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575667.606218:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1788575667.606221:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:1.0:1788575667.606222:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575667.606228:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575692 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788575667.606229:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788575667.606235:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.606242:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575667.606244:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575667.606245:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788575667.606255:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575667.606257:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575667.606261:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606266:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575667.606273:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575667.606273:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575667.606278:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307272121
00000100:00000040:1.0:1788575667.606279:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575667.606284:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:3.0:1788575667.606284:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000040:1.0:1788575667.606291:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12608)
00000100:00000001:3.0:1788575667.606291:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575667.606297:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307272121
00000100:00000010:3.0:1788575667.606300:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575667.606303:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575667.606305:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575667.606307:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.606309:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575667.606313:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575667/real 1788575667]  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575667.606319:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575667.606321:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575667.606322:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575667.606324:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575667.606325:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575667.606327:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575667.606329:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575667.606334:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606338:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575667.606339:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606340:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575667.606341:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575667.606344:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606349:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575667.606351:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606355:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575667.606357:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606358:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575667.606359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606360:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575667.606362:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575667.606363:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575667.606366:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575667.606367:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575667.606369:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575667.606373:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393226624:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575667.606377:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606379:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575667.606382:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393226624/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606386:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575667.606388:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575667.606390:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575667.606407:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575667.606408:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575667.606409:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575667.606410:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575667.606412:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:3.0:1788575667.606415:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575667.606416:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575667.606418:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:3.0:1788575667.606419:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575667.606420:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606422:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606427:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575667.606429:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.606430:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575667.606431:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575667.606432:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575667.606434:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606439:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575667.606440:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575667.606441:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575667.606443:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575667.606444:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575667.606445:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226752:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575667.606447:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575667.606448:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575667.606450:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575667.606451:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575667.606452:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575667.606454:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575667.606456:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575667.606458:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575667.606460:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575667.606462:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393226752, portal 10
00000100:00000001:3.0:1788575667.606463:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575667.606464:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345473920 : -130483364077696 : ffff8953796f5f80)
00000100:00000040:3.0:1788575667.606469:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606472:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575667.606474:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575667.606476:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575667.606477:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393226752, offset 0
00000400:00000200:3.0:1788575667.606479:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575667.606482:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575667.606484:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575667.606486:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575667.606488:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575667.606490:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575667.606491:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575667.606493:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331027:331027:256:4294967295] 192.168.202.110@tcp LPNI seq info [330984:330984:8:4294967295]
00000400:00000200:3.0:1788575667.606495:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575667.606499:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575667.606502:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575667.606503:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575667.606506:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575667.606508:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788575667.606510:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575667.606512:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1788575667.606514:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575667.606516:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575667.606517:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575667.606519:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575667.606522:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575667.606524:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575667.606526:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575667.606530:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606537:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575667.606539:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575692 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606546:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.606547:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575667.606551:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575667.606554:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575667.606557:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575667.606559:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307271207
00000100:00000001:3.0:1788575667.606561:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307272121
00000100:00000001:3.0:1788575667.606563:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307271207
00000100:00000001:3.0:1788575667.606565:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307272121
00000100:00000001:3.0:1788575667.606567:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575667.606569:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.606571:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575667.606574:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575667/real 1788575667]  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575667.606581:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575667.606582:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575667.606584:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575667.606586:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575667.606588:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575667.606590:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575667.606592:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575667.606601:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606607:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575667.606608:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606609:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575667.606611:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575667.606614:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606618:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575667.606620:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606625:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575667.606626:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606627:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575667.606628:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606629:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575667.606630:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575667.606632:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575667.606649:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575667.606652:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575667.606656:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575667.606662:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226752:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575667.606665:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575667.606668:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606674:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575667.606675:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575667.606677:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575667.606679:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575667.606681:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575667.606683:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226880:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575667.606686:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575667.606688:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575667.606690:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575667.606691:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575667.606695:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575667.606697:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575667.606700:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575667.606703:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575667.606707:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575667.606709:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393226880, portal 4
00000100:00000001:3.0:1788575667.606710:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575667.606711:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345465856 : -130483364085760 : ffff8953796f4000)
00000100:00000040:3.0:1788575667.606715:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606719:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575667.606721:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575667.606722:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575667.606723:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393226880, offset 0
00000400:00000200:3.0:1788575667.606725:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575667.606728:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575667.606730:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575667.606732:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575667.606733:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575667.606735:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575667.606736:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575667.606738:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331028:331028:256:4294967295] 192.168.202.110@tcp LPNI seq info [330985:330985:8:4294967295]
00000400:00000200:3.0:1788575667.606740:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575667.606744:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575667.606747:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575667.606749:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575667.606751:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575667.606753:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788575667.606756:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575667.606758:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1788575667.606760:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575667.606761:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575667.606763:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575667.606765:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575667.606767:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575667.606768:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575667.606770:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575667.606772:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606777:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575667.606779:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575692 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606782:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.606783:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575667.606785:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575667.606787:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575667.606789:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575667.606790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307271353
00000100:00000001:3.0:1788575667.606791:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307272121
00000100:00000001:3.0:1788575667.606793:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307271353
00000100:00000001:3.0:1788575667.606794:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307272121
00000100:00000001:3.0:1788575667.606795:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575667.606796:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.606797:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575667.606799:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575667/real 1788575667]  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575667.606804:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575667.606804:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575667.606805:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575667.606807:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575667.606808:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575667.606809:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575667.606811:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575667.606817:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606828:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575667.606830:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606831:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575667.606833:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575667.606836:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606842:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575667.606846:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606857:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575667.606859:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606861:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575667.606862:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606864:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575667.606865:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575667.606867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575667.606870:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575667.606873:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575667.606876:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575667.606883:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393226880:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575667.606887:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606888:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575667.606891:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5f80 x1875444393226752/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606897:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575667.606898:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575667.606901:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575667.606903:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575667.606905:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575667.606906:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575667.606908:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575667.606910:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:3.0:1788575667.606912:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575667.606913:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575667.606914:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5f80.
00000100:00000001:3.0:1788575667.606917:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575667.606918:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606920:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575667.606923:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4000 x1875444393226880/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575692 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575667.606929:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575667.606930:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575667.606931:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575667.606932:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575667.606933:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575667.606934:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575667.606935:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575667.606936:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:3.0:1788575667.606937:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575667.606938:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575667.606939:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4000.
00000100:00000001:3.0:1788575667.606940:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575667.606941:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606942:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575667.606944:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575667.606945:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575667.606946:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575667.606947:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575667.989510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575667.989517:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575667.989520:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575667.989522:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575667.989525:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575667.989527:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575667.989529:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575667.989530:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575667.989533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575667.989535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575667.989537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575667.989537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575667.989604:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575667.989608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575667.989611:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575668.117483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575668.117490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575668.117492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575668.117497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575668.117501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575668.117504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575668.117507:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575668.117509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575668.117512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575668.565564:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307272121, at 4307272128
00000100:00000010:1.0:1788575668.565580:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000040:0.0:1788575668.565759:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307272121, at 4307272128
00000100:00000010:0.0:1788575668.565766:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00010000:00000001:3.0:1788575669.016763:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575669.016768:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575669.016771:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575669.016772:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575669.016774:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575669.016775:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575669.016896:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575669.016899:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575669.016903:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575669.141500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575669.141508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575669.141509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575669.141515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575669.141518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575669.141522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575669.141524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575669.141526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575669.141529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575670.037463:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575670.037469:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575670.037473:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575670.037474:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575670.037478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575670.037479:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575670.037526:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575670.037529:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575670.037531:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575670.165508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575670.165513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575670.165515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575670.165519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575670.165521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575670.165524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575670.165526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575670.165528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575670.165530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575671.061503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575671.061508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575671.061514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575671.061515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575671.061518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575671.061519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575671.061572:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575671.061575:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575671.061578:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575671.189503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575671.189509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575671.189510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575671.189514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575671.189517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575671.189521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575671.189523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575671.189526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575671.189529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575672.085482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575672.085488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575672.085493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575672.085494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575672.085497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575672.085499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575672.085536:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575672.085539:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575672.085541:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575672.213520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575672.213527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575672.213528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575672.213534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575672.213537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575672.213541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575672.213543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575672.213546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575672.213550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575672.725498:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575672.725508:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575672.725512:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575672.725516:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575672.725521:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575672.725529:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12603
00000400:00000200:1.0:1788575672.725541:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575672.725548:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575672.725551:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575672.725554:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725558:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575672.725560:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725563:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575672.725567:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575672.725569:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575672.725571:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575672.725575:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575672.725577:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575672.725579:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575672.725582:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725584:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575672.725586:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725588:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575672.725591:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575672.725594:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725597:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575672.725600:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575672.725603:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575672.725606:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575672.725608:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575672.725610:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575672.725619:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788575672.725622:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575672.725631:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575672.725655:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575672.725658:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575672.725661:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575672.725665:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575672.725673:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788575672.725679:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575672.725688:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575672.725698:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575672.725701:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575672.725705:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575672.725729:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575672.725731:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575672.725733:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575672.725734:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575672.725735:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575672.725737:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575672.725739:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12603
00000400:00000200:1.0:1788575672.725742:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575672.725745:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575672.725747:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575672.725749:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725750:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575672.725752:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725753:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575672.725754:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1788575672.725755:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575672.725756:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575672.725759:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575672.725760:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575672.725760:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1788575672.725764:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575672.725764:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1788575672.725768:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575672.725772:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725773:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575672.725774:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575672.725775:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725779:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575672.725785:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575672.725791:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575672.725792:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575672.725794:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575672.725796:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575672.725798:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575672.725800:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575672.725803:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575672.725806:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575672.725806:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575672.725812:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575672.725813:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393227008:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788575672.725816:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575672.725819:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:1.0:1788575672.725822:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575672.725824:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575672.725827:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1788575672.725828:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575672.725830:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788575672.725832:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575672.725837:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
00000100:00000010:1.0:1788575672.725838:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
02000000:00000001:3.0:1788575672.725840:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788575672.725843:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575672.725848:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1788575672.725848:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575672.725851:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575672.725854:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1788575672.725855:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
02000000:00000001:1.0:1788575672.725856:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575672.725860:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575672.725863:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000010:1.0:1788575672.725865:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788575672.725870:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575672.725871:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393227008, portal 25
00000100:00000001:3.0:1788575672.725875:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00100000:1.0:1788575672.725878:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575672.725878:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345466752 : -130483364084864 : ffff8953796f4380)
00000100:00000040:3.0:1788575672.725887:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575672.725889:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575672.725894:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575672.725900:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.725900:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575672.725913:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725914:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575672.725917:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00080000:1.0:1788575672.725918:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575672.725922:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575672.725923:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:3.0:1788575672.725924:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393227008, offset 0
00000100:00080000:1.0:1788575672.725925:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575672.725929:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575672.725934:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12603
00000400:00000200:3.0:1788575672.725935:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788575672.725939:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575672.725942:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575672.725945:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1788575672.725945:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788575672.725949:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725951:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575672.725956:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575672.725956:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:1.0:1788575672.725957:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575672.725961:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575672.725962:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575672.725963:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1788575672.725964:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:1.0:1788575672.725965:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575672.725969:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575672.725970:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:3.0:1788575672.725972:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:1.0:1788575672.725976:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575672.725980:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575672.725983:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575672.725983:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1788575672.725989:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575672.725989:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:1.0:1788575672.725994:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575672.725997:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331029:331029:256:4294967295] 192.168.202.110@tcp LPNI seq info [330986:330986:8:4294967295]
00000100:00080000:1.0:1788575672.726000:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575672.726005:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575672.726006:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575672.726009:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575672.726013:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575672.726018:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575672.726019:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1788575672.726021:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575672.726025:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:3.0:1788575672.726029:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00000001:1.0:1788575672.726031:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575672.726034:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575672.726036:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000100:00000010:1.0:1788575672.726039:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000800:00000200:3.0:1788575672.726044:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:1.0:1788575672.726045:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575672.726050:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.0:1788575672.726050:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
02000000:00000001:1.0:1788575672.726053:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575672.726056:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:3.1:1788575672.726058:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1788575672.726059:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575672.726063:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1788575672.726064:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
02000000:00000010:1.0:1788575672.726070:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000800:00000100:3.0:1788575672.726070:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575672.726072:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000100:00000001:1.0:1788575672.726076:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575672.726079:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:1.0:1788575672.726084:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575672.726086:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575672.726092:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575672.726096:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:1.0:1788575672.726098:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575672.726101:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:1.0:1788575672.726102:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575672.726107:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575672.726109:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575672.726129:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575672.726132:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575672.726132:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575672.726136:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575672.726138:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575672.726140:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12613)
00000100:00000001:3.0:1788575672.726149:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575672.726151:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575672.726157:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575672.726161:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575672.726164:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307277241
00000100:00000010:3.0:1788575672.726168:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575672.726174:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575672.726179:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307277241
00000100:00000010:3.0:1788575672.726181:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575672.726184:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575672.726187:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575672.726190:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575672.726192:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575672.726196:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575672/real 1788575672]  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575672.726218:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575672.726220:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575672.726222:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575672.726224:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575672.726226:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575672.726228:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575672.726231:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575672.726239:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726246:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575672.726248:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726250:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575672.726252:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575672.726256:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726262:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575672.726266:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726273:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575672.726275:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726277:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575672.726278:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726280:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575672.726282:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575672.726285:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575672.726288:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575672.726290:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575672.726294:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575672.726303:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393227008:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575672.726307:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726309:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575672.726312:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4380 x1875444393227008/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726318:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575672.726320:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575672.726322:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575672.726325:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575672.726327:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575672.726328:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575672.726330:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575672.726332:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:3.0:1788575672.726335:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575672.726336:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575672.726338:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4380.
00000100:00000001:3.0:1788575672.726341:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575672.726342:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726344:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726351:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575672.726353:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575672.726354:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575672.726356:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575672.726357:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575672.726360:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726367:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575672.726368:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575672.726370:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575672.726372:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575672.726374:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575672.726376:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227136:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575672.726380:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575672.726381:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575672.726383:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575672.726385:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575672.726387:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575672.726389:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575672.726409:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575672.726411:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575672.726415:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575672.726418:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393227136, portal 10
00000100:00000001:3.0:1788575672.726419:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575672.726421:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345468544 : -130483364083072 : ffff8953796f4a80)
00000100:00000040:3.0:1788575672.726427:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726433:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575672.726436:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575672.726438:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575672.726440:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393227136, offset 0
00000400:00000200:3.0:1788575672.726444:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575672.726447:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575672.726450:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575672.726453:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575672.726455:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575672.726458:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575672.726460:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575672.726463:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331030:331030:256:4294967295] 192.168.202.110@tcp LPNI seq info [330987:330987:8:4294967295]
00000400:00000200:3.0:1788575672.726467:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575672.726474:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575672.726477:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575672.726480:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575672.726483:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575672.726486:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1788575672.726490:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575672.726493:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1788575672.726496:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575672.726498:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575672.726500:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575672.726504:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575672.726506:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575672.726508:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575672.726510:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575672.726513:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726521:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575672.726524:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726530:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575672.726531:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575672.726534:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575672.726536:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575672.726539:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575672.726541:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307276327
00000100:00000001:3.0:1788575672.726544:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307277241
00000100:00000001:3.0:1788575672.726546:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307276327
00000100:00000001:3.0:1788575672.726548:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307277241
00000100:00000001:3.0:1788575672.726550:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575672.726551:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575672.726553:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575672.726556:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575672/real 1788575672]  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575672.726563:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575672.726564:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575672.726566:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575672.726568:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575672.726570:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575672.726571:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575672.726574:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575672.726582:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726588:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575672.726589:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726590:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575672.726592:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575672.726595:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726601:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575672.726604:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726611:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575672.726613:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726615:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575672.726616:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726617:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575672.726619:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575672.726622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575672.726624:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575672.726626:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575672.726630:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575672.726652:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227136:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575672.726656:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575672.726659:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726666:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575672.726667:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575672.726669:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575672.726671:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575672.726673:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575672.726675:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575672.726678:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575672.726680:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575672.726681:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575672.726682:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575672.726686:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575672.726688:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575672.726691:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575672.726693:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575672.726696:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575672.726698:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393227264, portal 4
00000100:00000001:3.0:1788575672.726700:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575672.726702:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345476608 : -130483364075008 : ffff8953796f6a00)
00000100:00000040:3.0:1788575672.726707:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726713:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575672.726715:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575672.726717:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575672.726719:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393227264, offset 0
00000400:00000200:3.0:1788575672.726723:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575672.726726:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575672.726729:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575672.726731:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575672.726734:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575672.726736:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575672.726738:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575672.726741:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331031:331031:256:4294967295] 192.168.202.110@tcp LPNI seq info [330988:330988:8:4294967295]
00000400:00000200:3.0:1788575672.726745:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575672.726751:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575672.726755:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575672.726757:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575672.726761:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575672.726763:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1788575672.726767:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575672.726769:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1788575672.726773:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575672.726774:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575672.726777:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575672.726780:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575672.726783:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575672.726784:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575672.726786:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575672.726790:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726797:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575672.726799:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726806:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575672.726807:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575672.726810:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575672.726812:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575672.726815:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575672.726817:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307276473
00000100:00000001:3.0:1788575672.726819:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307277241
00000100:00000001:3.0:1788575672.726821:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307276473
00000100:00000001:3.0:1788575672.726823:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307277241
00000100:00000001:3.0:1788575672.726825:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575672.726827:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575672.726829:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575672.726832:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575672/real 1788575672]  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575672.726839:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575672.726840:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575672.726841:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575672.726843:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575672.726845:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575672.726846:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575672.726849:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575672.726857:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726863:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575672.726864:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726866:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575672.726867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575672.726870:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726877:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575672.726880:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726886:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575672.726888:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726890:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575672.726891:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726893:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575672.726894:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575672.726897:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575672.726900:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575672.726902:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575672.726905:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575672.726912:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575672.726915:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726917:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575672.726920:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4a80 x1875444393227136/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726926:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575672.726927:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575672.726929:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575672.726931:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575672.726932:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575672.726933:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575672.726935:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575672.726936:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:3.0:1788575672.726939:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575672.726940:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575672.726941:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4a80.
00000100:00000001:3.0:1788575672.726943:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575672.726944:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726946:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575672.726948:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6a00 x1875444393227264/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575672.726954:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575672.726955:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575672.726956:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575672.726959:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575672.726960:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575672.726961:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575672.726963:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575672.726964:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:3.0:1788575672.726966:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575672.726967:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575672.726968:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6a00.
00000100:00000001:3.0:1788575672.726970:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575672.726971:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726973:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575672.726976:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575672.726977:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575672.726979:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575672.726981:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575673.109515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575673.109521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575673.109524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575673.109525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575673.109527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575673.109528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575673.109577:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575673.109580:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575673.109583:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575673.237596:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575673.237604:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575673.237606:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575673.237612:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575673.237616:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575673.237620:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575673.237623:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575673.237626:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575673.237629:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575673.685526:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307277241, at 4307277248
00000100:00000040:1.0:1788575673.685526:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307277241, at 4307277248
00000100:00000010:0.0:1788575673.685539:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:1.0:1788575673.685542:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000800:00000200:3.0:1788575673.749539:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1788575673.749548:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1788575673.749553:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:0.0:1788575673.749725:0:2355:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788575673.749731:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575673.749744:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788575673.749749:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575673.749752:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575674.133527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575674.133536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575674.133542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575674.133543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575674.133547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575674.133549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575674.133598:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575674.133604:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575674.133607:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575674.261508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575674.261514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575674.261516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575674.261522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575674.261525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575674.261528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575674.261530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575674.261533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575674.261536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575675.157510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575675.157517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575675.157522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575675.157525:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575675.157527:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575675.157531:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575675.157533:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575675.157536:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575675.157538:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575675.157540:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575675.157543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575675.157545:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575675.157546:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575675.157547:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575675.157548:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575675.157549:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575675.157550:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575675.157552:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575675.157610:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575675.157612:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575675.157614:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575675.285473:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575675.285480:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575675.285482:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575675.285487:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575675.285490:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575675.285493:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575675.285496:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575675.285498:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575675.285501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575676.181555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575676.181563:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575676.181569:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575676.181571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575676.181575:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575676.181576:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575676.181717:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575676.181721:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575676.181724:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575676.309589:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575676.309599:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575676.309602:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575676.309609:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575676.309615:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575676.309620:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575676.309623:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575676.309626:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575676.309629:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575677.205553:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575677.205563:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575677.205570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575677.205572:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575677.205576:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575677.205578:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575677.205668:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575677.205672:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575677.205676:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575677.333564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575677.333572:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575677.333575:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575677.333582:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575677.333586:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575677.333591:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.333594:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575677.333597:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.333601:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575677.845680:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575677.845693:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575677.845696:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575677.845701:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575677.845706:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575677.845715:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12608
00000400:00000200:3.0:1788575677.845727:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575677.845734:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575677.845737:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575677.845740:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.845744:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575677.845746:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.845749:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575677.845753:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575677.845755:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575677.845759:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575677.845762:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575677.845764:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575677.845766:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575677.845769:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.845771:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575677.845772:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.845774:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575677.845777:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575677.845780:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575677.845782:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575677.845786:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575677.845789:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575677.845792:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575677.845795:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575677.845797:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575677.845809:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575677.845813:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575677.845818:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575677.845820:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575677.845823:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575677.845826:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575677.845831:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575677.845839:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575677.845847:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575677.845858:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575677.845868:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575677.845874:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575677.845879:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575677.845908:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575677.845912:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575677.845916:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575677.845918:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575677.845919:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575677.845922:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575677.845925:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12608
00000400:00000200:3.0:1788575677.845929:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575677.845933:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575677.845936:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575677.845938:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.845940:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575677.845942:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.845943:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575677.845945:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575677.845946:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575677.845947:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575677.845949:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575677.845950:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575677.845958:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575677.845962:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575677.845965:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575677.845969:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575677.845969:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575677.845974:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575677.845974:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.845979:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575677.845985:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:2.0:1788575677.845986:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575677.845990:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575677.845996:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575677.846001:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575677.846004:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575677.846007:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575677.846010:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575677.846012:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575677.846015:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575677.846017:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575677.846019:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575677.846023:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575677.846027:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00100000:2.0:1788575677.846030:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393227392:192.168.202.110@tcp:250:kworker.0
00000008:00000001:3.0:1788575677.846032:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575677.846037:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.846039:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:3.0:1788575677.846043:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
02000000:00000001:2.0:1788575677.846046:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:3.0:1788575677.846048:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575677.846050:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575677.846054:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575677.846055:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1788575677.846057:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575677.846061:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575677.846062:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:2.0:1788575677.846064:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575677.846065:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575677.846069:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575677.846073:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000400:00000200:2.0:1788575677.846079:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575677.846080:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575677.846085:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00100000:3.0:1788575677.846090:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1788575677.846095:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000001:3.0:1788575677.846097:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575677.846100:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575677.846103:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575677.846106:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393227392, portal 25
00000100:00000001:3.0:1788575677.846110:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.846112:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1788575677.846113:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575677.846115:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00080000:3.0:1788575677.846117:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575677.846120:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575677.846122:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575677.846124:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000040:2.0:1788575677.846127:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575677.846128:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12608
00000400:00000200:3.0:1788575677.846134:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575677.846139:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575677.846143:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575677.846146:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575677.846147:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.846151:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575677.846154:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575677.846156:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575677.846157:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:2.0:1788575677.846161:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000020:00000001:3.0:1788575677.846161:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575677.846165:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575677.846167:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:2.0:1788575677.846168:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393227392, offset 0
00000020:00000040:3.0:1788575677.846170:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575677.846174:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575677.846178:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575677.846180:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575677.846186:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.846189:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575677.846192:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575677.846196:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575677.846197:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575677.846202:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575677.846208:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:3.0:1788575677.846209:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575677.846215:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575677.846218:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1788575677.846218:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575677.846222:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575677.846226:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1788575677.846227:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575677.846231:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788575677.846234:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00100000:3.0:1788575677.846235:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575677.846239:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575677.846240:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575677.846242:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575677.846246:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000400:00000200:2.0:1788575677.846247:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331032:331032:256:4294967295] 192.168.202.110@tcp LPNI seq info [330989:330989:8:4294967295]
00000020:00000040:3.0:1788575677.846250:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575677.846255:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575677.846258:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575677.846258:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575677.846262:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575677.846264:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575677.846265:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575677.846268:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575677.846274:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575677.846277:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00100000:3.0:1788575677.846283:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1788575677.846290:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575677.846299:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000100:00000001:3.0:1788575677.846299:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575677.846304:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575677.846311:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575677.846312:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788575677.846319:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:3.0:1788575677.846324:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575677.846329:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.1:1788575677.846329:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575677.846334:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:3.0:1788575677.846336:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575677.846344:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12619)
00000800:00000100:2.0:1788575677.846352:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575677.846356:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575677.846365:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575677.846371:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575677.846377:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575677.846381:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575677.846387:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575677.846427:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846448:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575677.846452:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846459:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.846460:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575677.846467:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575677.846472:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575677.846475:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307282362
00000100:00000010:2.0:1788575677.846482:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cc60.
00000100:00000001:2.0:1788575677.846489:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575677.846496:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307282362
00000100:00000010:2.0:1788575677.846499:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cba0.
00000100:00000001:2.0:1788575677.846502:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575677.846505:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575677.846508:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.846510:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575677.846515:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575677/real 1788575677]  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575677.846525:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575677.846527:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575677.846529:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575677.846532:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575677.846534:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575677.846536:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575677.846540:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575677.846549:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846564:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575677.846567:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.846570:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575677.846572:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575677.846576:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846582:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575677.846587:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846594:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575677.846597:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.846598:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575677.846599:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.846602:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575677.846604:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575677.846607:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575677.846613:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575677.846615:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575677.846620:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575677.846628:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393227392:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575677.846645:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.846647:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575677.846651:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393227392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846658:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575677.846661:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575677.846664:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575677.846668:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575677.846669:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575677.846672:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575677.846674:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575677.846677:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575677.846681:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575677.846683:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575677.846685:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575677.846688:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575677.846690:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.846692:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.846701:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575677.846702:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.846704:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575677.846706:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575677.846707:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575677.846710:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846717:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575677.846718:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575677.846720:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575677.846722:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575677.846724:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575677.846727:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575677.846730:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575677.846732:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575677.846733:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575677.846736:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575677.846738:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575677.846740:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575677.846744:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575677.846747:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575677.846750:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575677.846753:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393227520, portal 10
00000100:00000001:2.0:1788575677.846755:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575677.846756:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:2.0:1788575677.846763:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846770:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575677.846772:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575677.846774:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575677.846777:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393227520, offset 0
00000400:00000200:2.0:1788575677.846780:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575677.846784:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575677.846788:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575677.846791:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575677.846793:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575677.846795:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575677.846797:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575677.846800:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331033:331033:256:4294967295] 192.168.202.110@tcp LPNI seq info [330990:330990:8:4294967295]
00000400:00000200:2.0:1788575677.846804:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575677.846810:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575677.846814:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575677.846818:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575677.846822:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575677.846825:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575677.846829:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575677.846831:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575677.846836:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575677.846837:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575677.846840:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575677.846844:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575677.846846:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575677.846848:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575677.846850:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575677.846853:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846860:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575677.846862:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846874:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.846876:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575677.846879:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575677.846883:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575677.846887:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575677.846889:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307281447
00000100:00000001:2.0:1788575677.846892:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307282362
00000100:00000001:2.0:1788575677.846894:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307281447
00000100:00000001:2.0:1788575677.846896:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307282362
00000100:00000001:2.0:1788575677.846898:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575677.846900:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.846902:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575677.846906:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575677/real 1788575677]  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575677.846914:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575677.846915:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575677.846917:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575677.846919:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575677.846921:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575677.846923:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575677.846926:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575677.846934:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846940:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575677.846941:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.846943:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575677.846944:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575677.846948:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846955:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575677.846958:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.846965:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575677.846967:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.846969:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575677.846970:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.846972:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575677.846974:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575677.846977:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575677.846981:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575677.846983:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575677.846988:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575677.846995:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575677.846999:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575677.847004:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.847010:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575677.847011:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575677.847013:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575677.847015:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575677.847017:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575677.847020:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227648:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575677.847023:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575677.847025:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575677.847027:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575677.847029:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575677.847032:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575677.847034:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575677.847037:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575677.847040:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575677.847044:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575677.847047:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393227648, portal 4
00000100:00000001:2.0:1788575677.847049:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575677.847050:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:2.0:1788575677.847057:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.847063:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575677.847065:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575677.847067:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575677.847070:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393227648, offset 0
00000400:00000200:2.0:1788575677.847074:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575677.847077:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575677.847081:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575677.847084:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575677.847086:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575677.847088:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575677.847091:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575677.847094:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331034:331034:256:4294967295] 192.168.202.110@tcp LPNI seq info [330991:330991:8:4294967295]
00000400:00000200:2.0:1788575677.847098:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575677.847103:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575677.847108:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575677.847111:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575677.847115:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575677.847117:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575677.847121:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575677.847124:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575677.847127:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575677.847129:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575677.847132:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575677.847135:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575677.847138:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575677.847139:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575677.847142:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575677.847145:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.847152:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575677.847155:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.847160:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.847162:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575677.847165:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575677.847167:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575677.847170:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575677.847172:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307281593
00000100:00000001:2.0:1788575677.847174:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307282362
00000100:00000001:2.0:1788575677.847176:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307281593
00000100:00000001:2.0:1788575677.847178:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307282362
00000100:00000001:2.0:1788575677.847180:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575677.847182:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.847184:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575677.847187:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575677/real 1788575677]  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575677.847194:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575677.847196:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575677.847197:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575677.847199:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575677.847200:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575677.847203:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575677.847206:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575677.847213:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.847219:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575677.847221:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.847223:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575677.847224:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575677.847227:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.847234:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575677.847238:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.847244:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575677.847246:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.847248:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575677.847249:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.847251:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575677.847252:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575677.847255:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575677.847258:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575677.847260:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575677.847263:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575677.847271:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227648:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575677.847274:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.847276:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575677.847279:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393227520/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.847285:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575677.847287:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575677.847289:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575677.847292:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575677.847293:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575677.847294:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575677.847296:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575677.847298:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575677.847300:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575677.847302:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575677.847303:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788575677.847305:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575677.847307:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.847308:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575677.847311:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393227648/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575677.847317:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575677.847318:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575677.847320:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575677.847322:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575677.847323:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575677.847324:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575677.847326:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575677.847327:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575677.847329:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575677.847330:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575677.847331:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788575677.847334:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575677.847335:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.847337:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575677.847340:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575677.847341:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575677.847343:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575677.847345:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575678.229471:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575678.229480:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575678.229482:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575678.229485:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575678.229490:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575678.229492:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575678.229494:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575678.229497:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575678.229501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575678.229502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575678.229505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575678.229506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575678.229583:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575678.229588:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575678.229597:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575678.357591:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575678.357602:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575678.357605:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575678.357611:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575678.357615:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575678.357620:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575678.357623:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575678.357626:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575678.357630:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575678.805491:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307282362, at 4307282368
00000100:00000040:1.0:1788575678.805491:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307282362, at 4307282368
00000100:00000010:1.0:1788575678.805511:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000100:00000010:0.0:1788575678.805513:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00010000:00000001:3.0:1788575679.253554:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575679.253563:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575679.253569:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575679.253571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575679.253575:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575679.253576:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575679.253629:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575679.253646:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575679.253652:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575679.381574:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575679.381584:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575679.381587:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575679.381596:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575679.381601:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575679.381605:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575679.381608:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575679.381611:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575679.381614:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575680.278300:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575680.278307:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575680.278312:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575680.278314:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575680.278317:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575680.278318:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575680.278411:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575680.278415:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575680.278419:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575680.407762:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575680.407768:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575680.407771:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575680.407776:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575680.407780:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575680.407784:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575680.407786:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575680.407789:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575680.407793:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575681.301488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575681.301495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575681.301500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575681.301502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575681.301505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575681.301507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575681.301560:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575681.301564:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575681.301567:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575681.431784:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575681.431794:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575681.431796:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575681.431802:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575681.431807:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575681.431810:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575681.431813:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575681.431816:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575681.431819:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575682.325513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575682.325518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575682.325522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575682.325523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575682.325525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575682.325526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575682.325594:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575682.325598:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575682.325602:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575682.453493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575682.453499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575682.453501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575682.453506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575682.453510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575682.453513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575682.453516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575682.453518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575682.453521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575682.965542:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575682.965563:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575682.965566:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575682.965570:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575682.965575:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575682.965583:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12614
00000400:00000200:0.0:1788575682.965592:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575682.965598:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575682.965601:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575682.965604:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.965607:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575682.965609:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.965612:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575682.965615:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575682.965617:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575682.965620:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575682.965623:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575682.965625:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575682.965627:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575682.965640:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.965642:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575682.965644:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.965646:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575682.965649:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575682.965653:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575682.965655:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575682.965658:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575682.965662:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575682.965664:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575682.965667:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575682.965669:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575682.965677:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788575682.965681:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575682.965684:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575682.965686:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575682.965689:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575682.965691:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575682.965695:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575682.965702:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788575682.965715:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575682.965725:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575682.965735:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575682.965740:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575682.965744:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575682.965764:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575682.965767:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575682.965770:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575682.965772:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575682.965773:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575682.965776:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575682.965778:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12614
00000400:00000200:0.0:1788575682.965782:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575682.965785:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575682.965788:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575682.965791:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.965794:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575682.965796:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.965798:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575682.965800:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575682.965802:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575682.965802:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788575682.965806:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575682.965808:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1788575682.965809:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575682.965812:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1788575682.965813:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575682.965818:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575682.965822:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575682.965823:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.965828:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575682.965832:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.965837:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575682.965842:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1788575682.965843:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575682.965846:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575682.965848:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575682.965851:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575682.965852:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575682.965855:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575682.965856:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575682.965860:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575682.965865:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393227776:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788575682.965869:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575682.965870:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575682.965875:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575682.965876:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1788575682.965878:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575682.965880:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1788575682.965883:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575682.965885:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:0.0:1788575682.965889:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575682.965892:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575682.965892:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575682.965896:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575682.965897:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788575682.965901:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:2.0:1788575682.965905:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575682.965906:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575682.965910:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575682.965911:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575682.965913:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575682.965914:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575682.965918:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000400:00000010:2.0:1788575682.965920:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00000001:0.0:1788575682.965922:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575682.965928:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00100000:0.0:1788575682.965930:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788575682.965936:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393227776, portal 25
00000100:00000001:2.0:1788575682.965941:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575682.965943:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523503232 : -130484186048384 : ffff895348711880)
00000100:00000001:0.0:1788575682.965948:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1788575682.965951:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575682.965956:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.965965:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575682.965966:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575682.965971:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575682.965975:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575682.965981:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393227776, offset 0
00000100:00000001:0.0:1788575682.965983:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575682.965987:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575682.965991:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788575682.965994:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575682.965997:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575682.965999:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575682.966002:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575682.966004:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575682.966009:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12614
00000400:00000200:2.0:1788575682.966011:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575682.966018:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575682.966019:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575682.966026:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575682.966027:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1788575682.966032:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575682.966034:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1788575682.966039:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575682.966040:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.966045:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575682.966046:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331035:331035:256:4294967295] 192.168.202.110@tcp LPNI seq info [330992:330992:8:4294967295]
00000100:00000040:0.0:1788575682.966051:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575682.966056:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575682.966060:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575682.966065:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575682.966069:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:2.0:1788575682.966070:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000001:0.0:1788575682.966072:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575682.966076:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:2.0:1788575682.966080:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788575682.966081:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575682.966085:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000010:2.0:1788575682.966087:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000100:00000040:0.0:1788575682.966091:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.966094:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1788575682.966098:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788575682.966099:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575682.966104:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.0:1788575682.966104:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000040:0.0:1788575682.966108:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:2.1:1788575682.966114:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1788575682.966116:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:2.1:1788575682.966119:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1788575682.966121:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575682.966126:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000100:2.0:1788575682.966129:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575682.966130:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575682.966135:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575682.966135:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000100:00000001:0.0:1788575682.966140:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575682.966143:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575682.966144:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00000001:0.0:1788575682.966148:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575682.966152:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575682.966153:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000010:0.0:1788575682.966160:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000400:00000200:2.0:1788575682.966161:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000020:00000040:0.0:1788575682.966165:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000010:2.0:1788575682.966166:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
02000000:00000001:0.0:1788575682.966173:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575682.966174:0:2361:0:(events.c:35:request_out_callback()) Process entered
02000000:00000001:0.0:1788575682.966175:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575682.966179:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575682.966181:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1788575682.966182:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575682.966183:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575682.966191:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575682.966197:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.966197:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575682.966203:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575707 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575682.966206:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575682.966218:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.966220:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575682.966224:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575682.966232:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575682.966233:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575682.966236:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307287481
00000100:00000001:0.0:1788575682.966237:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575682.966244:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c840.
00000100:00000040:0.0:1788575682.966245:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575682.966252:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575682.966257:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575682.966262:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575682.966262:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:0.0:1788575682.966271:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12624)
00000100:00000001:2.0:1788575682.966271:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307287481
00000100:00000010:2.0:1788575682.966277:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c000.
00000100:00000001:2.0:1788575682.966281:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575682.966284:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575682.966287:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.966289:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575682.966295:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575682/real 1788575682]  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575682.966304:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575682.966306:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575682.966308:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575682.966311:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575682.966313:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575682.966315:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575682.966319:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575682.966328:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966335:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575682.966337:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966340:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575682.966342:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575682.966346:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966353:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575682.966357:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966364:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575682.966367:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575682.966371:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966373:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575682.966375:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575682.966378:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575682.966381:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575682.966384:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575682.966388:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575682.966429:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393227776:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575682.966434:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966436:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575682.966440:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711880 x1875444393227776/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966452:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575682.966454:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575682.966457:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575682.966460:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575682.966462:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575682.966464:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575682.966466:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575682.966468:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:2.0:1788575682.966472:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575682.966474:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575682.966475:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711880.
00000100:00000001:2.0:1788575682.966478:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575682.966480:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966482:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966489:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575682.966491:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.966493:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575682.966495:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575682.966496:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575682.966500:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966507:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575682.966509:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575682.966511:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575682.966513:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575682.966515:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575682.966518:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227904:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575682.966521:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575682.966524:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575682.966525:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575682.966527:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575682.966530:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575682.966532:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575682.966536:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575682.966539:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575682.966542:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575682.966545:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393227904, portal 10
00000100:00000001:2.0:1788575682.966547:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575682.966548:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523511296 : -130484186040320 : ffff895348713800)
00000100:00000040:2.0:1788575682.966555:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966562:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575682.966564:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575682.966566:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575682.966569:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393227904, offset 0
00000400:00000200:2.0:1788575682.966573:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575682.966577:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575682.966581:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575682.966584:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575682.966586:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575682.966589:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575682.966591:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575682.966594:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331036:331036:256:4294967295] 192.168.202.110@tcp LPNI seq info [330993:330993:8:4294967295]
00000400:00000200:2.0:1788575682.966598:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575682.966604:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575682.966609:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575682.966612:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575682.966616:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575682.966619:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575682.966624:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575682.966626:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575682.966642:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575682.966644:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575682.966647:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575682.966651:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575682.966654:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575682.966656:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575682.966659:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575682.966662:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966670:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575682.966673:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575707 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966679:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.966680:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575682.966683:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575682.966686:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575682.966690:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575682.966692:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307286567
00000100:00000001:2.0:1788575682.966694:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307287481
00000100:00000001:2.0:1788575682.966696:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307286567
00000100:00000001:2.0:1788575682.966698:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307287481
00000100:00000001:2.0:1788575682.966700:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575682.966701:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.966702:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575682.966705:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575682/real 1788575682]  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575682.966710:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575682.966711:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575682.966712:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575682.966714:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575682.966715:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575682.966716:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575682.966718:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575682.966723:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966731:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575682.966732:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966733:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575682.966734:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575682.966736:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966739:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575682.966741:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966745:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575682.966746:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966747:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575682.966748:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966749:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575682.966750:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575682.966751:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575682.966753:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575682.966755:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575682.966757:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575682.966761:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393227904:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575682.966763:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575682.966765:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966769:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575682.966770:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575682.966771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575682.966773:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575682.966774:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575682.966775:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228032:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575682.966777:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575682.966779:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575682.966779:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575682.966780:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575682.966783:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575682.966784:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575682.966786:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575682.966788:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575682.966790:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575682.966791:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393228032, portal 4
00000100:00000001:2.0:1788575682.966792:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575682.966793:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:2.0:1788575682.966797:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966801:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575682.966802:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575682.966804:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575682.966805:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393228032, offset 0
00000400:00000200:2.0:1788575682.966807:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575682.966810:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575682.966812:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575682.966814:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575682.966816:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575682.966817:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575682.966819:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575682.966821:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331037:331037:256:4294967295] 192.168.202.110@tcp LPNI seq info [330994:330994:8:4294967295]
00000400:00000200:2.0:1788575682.966823:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575682.966826:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575682.966829:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575682.966831:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575682.966833:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575682.966835:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575682.966838:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575682.966839:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575682.966842:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575682.966843:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575682.966845:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575682.966847:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575682.966849:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575682.966850:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575682.966851:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575682.966853:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966857:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575682.966859:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575707 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966863:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.966864:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575682.966866:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575682.966868:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575682.966870:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575682.966871:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307286713
00000100:00000001:2.0:1788575682.966872:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307287481
00000100:00000001:2.0:1788575682.966874:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307286713
00000100:00000001:2.0:1788575682.966875:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307287481
00000100:00000001:2.0:1788575682.966876:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575682.966877:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.966878:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575682.966880:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575682/real 1788575682]  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575682.966885:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575682.966886:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575682.966887:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575682.966888:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575682.966889:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575682.966889:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575682.966891:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575682.966895:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966899:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575682.966899:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966900:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575682.966901:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575682.966903:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966906:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575682.966908:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966911:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575682.966913:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966914:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575682.966914:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966916:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575682.966917:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575682.966918:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575682.966920:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575682.966921:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575682.966924:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575682.966928:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228032:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575682.966930:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966931:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575682.966933:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713800 x1875444393227904/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966936:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575682.966937:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575682.966938:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575682.966939:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575682.966940:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575682.966941:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575682.966942:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575682.966943:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788575682.966944:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575682.966945:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575682.966946:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713800.
00000100:00000001:2.0:1788575682.966947:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575682.966948:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966949:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575682.966951:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393228032/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575707 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575682.966954:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575682.966955:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575682.966956:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575682.966957:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575682.966957:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575682.966958:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575682.966959:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575682.966959:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788575682.966960:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575682.966962:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575682.966962:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:2.0:1788575682.966964:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575682.966964:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966965:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575682.966968:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575682.966969:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575682.966970:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575682.966971:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575683.349507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575683.349514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575683.349519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575683.349521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575683.349524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575683.349525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575683.349586:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575683.349590:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575683.349593:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575683.478432:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575683.478438:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575683.478440:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575683.478444:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575683.478447:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575683.478451:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575683.478454:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575683.478456:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575683.478459:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575683.925511:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307287481, at 4307287488
00000100:00000010:1.0:1788575683.925523:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000100:00000040:0.0:1788575683.925607:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307287481, at 4307287488
00000100:00000010:0.0:1788575683.925613:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00010000:00000001:3.0:1788575684.373517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575684.373523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575684.373528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575684.373530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575684.373533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575684.373534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575684.373587:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575684.373590:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575684.373594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575684.501552:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575684.501559:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575684.501562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575684.501568:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575684.501571:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575684.501575:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575684.501578:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575684.501581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575684.501584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575685.397714:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575685.397722:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575685.397727:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575685.397731:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575685.397733:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575685.397736:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575685.397740:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575685.397742:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575685.397744:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575685.397747:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575685.397751:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575685.397752:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575685.397754:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575685.397756:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575685.397757:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575685.397759:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575685.397760:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575685.397762:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575685.397826:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575685.397831:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575685.397834:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575685.525538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575685.525545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575685.525547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575685.525552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575685.525555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575685.525559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575685.525562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575685.525564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575685.525567:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575686.421514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575686.421520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575686.421529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575686.421531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575686.421535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575686.421536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575686.421593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575686.421596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575686.421598:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575686.549532:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575686.549538:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575686.549540:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575686.549545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575686.549548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575686.549551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575686.549553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575686.549555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575686.549558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575687.445504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575687.445511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575687.445517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575687.445518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575687.445521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575687.445523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575687.445578:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575687.445581:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575687.445585:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575687.573477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575687.573485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575687.573488:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575687.573494:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575687.573498:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575687.573501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575687.573504:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575687.573507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575687.573510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575688.085490:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575688.085500:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575688.085503:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575688.085507:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575688.085511:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575688.085518:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12619
00000400:00000200:0.0:1788575688.085527:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575688.085533:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575688.085536:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575688.085539:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085542:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575688.085544:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085546:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575688.085551:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575688.085553:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575688.085555:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575688.085558:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575688.085561:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575688.085563:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575688.085566:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085568:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575688.085569:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085571:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575688.085574:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575688.085578:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085587:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575688.085590:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575688.085593:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575688.085596:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575688.085599:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575688.085601:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575688.085610:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788575688.085613:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575688.085617:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575688.085619:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575688.085622:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575688.085624:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575688.085628:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575688.085658:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575688.085664:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575688.085673:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575688.085682:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575688.085686:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575688.085690:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575688.085720:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575688.085724:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575688.085727:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575688.085729:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575688.085730:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575688.085733:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575688.085735:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12619
00000400:00000200:0.0:1788575688.085739:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575688.085742:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575688.085744:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575688.085748:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085750:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575688.085753:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085755:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575688.085756:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575688.085757:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575688.085758:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575688.085759:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1788575688.085760:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575688.085764:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575688.085765:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1788575688.085766:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575688.085770:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575688.085771:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085775:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575688.085781:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575688.085784:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085788:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575688.085792:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575688.085797:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085800:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575688.085804:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575688.085805:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788575688.085809:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575688.085809:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575688.085815:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575688.085816:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575688.085818:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575688.085820:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:0.0:1788575688.085823:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575688.085825:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788575688.085829:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575688.085833:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393228160:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575688.085834:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575688.085840:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000100:00000001:2.0:1788575688.085841:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1788575688.085846:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575688.085848:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575688.085851:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575688.085852:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575688.085855:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575688.085857:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788575688.085858:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575688.085861:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575688.085864:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575688.085864:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575688.085869:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575688.085872:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000400:00000200:2.0:1788575688.085877:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575688.085878:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575688.085883:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00100000:0.0:1788575688.085885:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1788575688.085890:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000001:0.0:1788575688.085891:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575688.085894:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575688.085897:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575688.085897:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393228160, portal 25
00000100:00000001:2.0:1788575688.085902:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575688.085906:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00000001:0.0:1788575688.085908:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575688.085912:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575688.085914:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575688.085917:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:2.0:1788575688.085917:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788575688.085918:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575688.085922:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575688.085928:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12619
00000400:00000200:0.0:1788575688.085936:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575688.085943:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575688.085947:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575688.085947:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575688.085954:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575688.085954:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085959:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575688.085961:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000040:0.0:1788575688.085964:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575688.085967:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393228160, offset 0
00000100:00000001:0.0:1788575688.085968:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575688.085972:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575688.085973:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575688.085975:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575688.085976:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575688.085980:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575688.085981:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788575688.085982:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575688.085987:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085988:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575688.085989:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575688.085991:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575688.085994:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575688.085994:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788575688.085999:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575688.086001:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575688.086002:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575688.086004:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575688.086005:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1788575688.086006:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575688.086011:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575688.086013:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575688.086015:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00000001:0.0:1788575688.086016:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575688.086020:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575688.086022:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000400:00000200:2.0:1788575688.086023:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:0.0:1788575688.086026:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575688.086029:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575688.086030:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575688.086030:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788575688.086034:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575688.086036:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:0.0:1788575688.086037:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575688.086041:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575688.086045:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331038:331038:256:4294967295] 192.168.202.110@tcp LPNI seq info [330995:330995:8:4294967295]
02000000:00000010:0.0:1788575688.086046:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788575688.086050:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575688.086053:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:0.0:1788575688.086056:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575688.086067:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575688.086068:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788575688.086070:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575688.086076:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788575688.086078:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575688.086084:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000100:00000001:0.0:1788575688.086090:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575688.086092:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788575688.086095:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1788575688.086096:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:0.0:1788575688.086103:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:2.1:1788575688.086103:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575688.086108:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:0.0:1788575688.086112:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12629)
00000800:00000100:2.0:1788575688.086112:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575688.086117:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575688.086122:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575688.086125:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575688.086129:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575688.086131:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575688.086134:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575688.086138:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086144:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575688.086146:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575713 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086154:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575688.086156:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575688.086159:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575688.086162:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575688.086164:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307292601
00000100:00000010:2.0:1788575688.086167:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cf00.
00000100:00000001:2.0:1788575688.086172:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575688.086176:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307292601
00000100:00000010:2.0:1788575688.086177:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c600.
00000100:00000001:2.0:1788575688.086179:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575688.086181:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575688.086183:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575688.086185:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575688.086188:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575688/real 1788575688]  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575688.086194:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575688.086196:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575688.086198:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575688.086199:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575688.086200:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575688.086202:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575688.086204:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575688.086209:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086212:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575688.086213:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086215:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575688.086216:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575688.086218:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086223:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575688.086225:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086229:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575688.086231:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086232:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575688.086233:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086234:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575688.086235:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575688.086237:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575688.086239:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575688.086241:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575688.086243:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575688.086247:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393228160:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575688.086250:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086252:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575688.086253:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393228160/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086257:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575688.086259:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575688.086261:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575688.086263:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575688.086264:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575688.086265:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575688.086267:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575688.086269:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:2.0:1788575688.086271:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575688.086272:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575688.086274:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:2.0:1788575688.086276:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575688.086276:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086278:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086283:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575688.086286:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575688.086287:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575688.086289:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575688.086290:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575688.086293:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086300:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575688.086301:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575688.086303:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575688.086305:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575688.086307:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575688.086310:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575688.086313:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575688.086315:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575688.086317:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575688.086319:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575688.086321:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575688.086323:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575688.086327:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575688.086330:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575688.086333:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575688.086336:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393228288, portal 10
00000100:00000001:2.0:1788575688.086338:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575688.086340:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000040:2.0:1788575688.086346:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086352:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575688.086355:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575688.086357:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575688.086360:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393228288, offset 0
00000400:00000200:2.0:1788575688.086364:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575688.086368:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575688.086372:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575688.086375:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575688.086377:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575688.086380:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575688.086413:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575688.086417:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331039:331039:256:4294967295] 192.168.202.110@tcp LPNI seq info [330996:330996:8:4294967295]
00000400:00000200:2.0:1788575688.086422:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575688.086428:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575688.086433:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575688.086437:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575688.086440:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575688.086443:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575688.086448:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575688.086450:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575688.086454:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575688.086455:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575688.086458:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575688.086462:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575688.086465:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575688.086467:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575688.086470:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575688.086473:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086482:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575688.086485:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575713 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086492:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575688.086494:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575688.086497:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575688.086500:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575688.086503:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575688.086506:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307291687
00000100:00000001:2.0:1788575688.086508:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307292601
00000100:00000001:2.0:1788575688.086511:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307291687
00000100:00000001:2.0:1788575688.086513:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307292601
00000100:00000001:2.0:1788575688.086515:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575688.086517:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575688.086519:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575688.086522:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575688/real 1788575688]  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575688.086531:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575688.086532:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575688.086534:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575688.086537:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575688.086538:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575688.086540:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575688.086543:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575688.086551:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086557:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575688.086558:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086560:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575688.086561:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575688.086565:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086571:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575688.086575:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086581:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575688.086584:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086585:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575688.086586:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086588:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575688.086589:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575688.086591:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575688.086594:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575688.086596:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575688.086600:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575688.086606:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575688.086609:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575688.086612:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086618:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575688.086619:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575688.086621:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575688.086622:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575688.086624:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575688.086627:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575688.086646:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575688.086648:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575688.086650:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575688.086651:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575688.086655:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575688.086657:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575688.086660:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575688.086663:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575688.086667:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575688.086669:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393228416, portal 4
00000100:00000001:2.0:1788575688.086671:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575688.086672:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:2.0:1788575688.086678:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086684:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575688.086687:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575688.086689:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575688.086691:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393228416, offset 0
00000400:00000200:2.0:1788575688.086695:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575688.086698:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575688.086702:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575688.086704:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575688.086707:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575688.086709:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575688.086711:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575688.086714:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331040:331040:256:4294967295] 192.168.202.110@tcp LPNI seq info [330997:330997:8:4294967295]
00000400:00000200:2.0:1788575688.086717:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575688.086723:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575688.086727:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575688.086730:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575688.086733:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575688.086736:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575688.086740:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575688.086742:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575688.086745:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575688.086747:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575688.086750:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575688.086753:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575688.086756:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575688.086757:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575688.086760:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575688.086763:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086770:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575688.086773:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575713 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086778:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575688.086780:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575688.086783:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575688.086785:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575688.086788:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575688.086790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307291833
00000100:00000001:2.0:1788575688.086792:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307292601
00000100:00000001:2.0:1788575688.086794:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307291833
00000100:00000001:2.0:1788575688.086796:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307292601
00000100:00000001:2.0:1788575688.086798:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575688.086800:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575688.086802:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575688.086805:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575688/real 1788575688]  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575688.086813:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575688.086814:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575688.086815:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575688.086818:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575688.086819:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575688.086821:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575688.086824:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575688.086832:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086841:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575688.086842:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086843:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575688.086844:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575688.086846:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086850:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575688.086853:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086859:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575688.086861:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086863:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575688.086864:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086866:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575688.086867:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575688.086870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575688.086873:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575688.086875:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575688.086879:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575688.086886:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575688.086889:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086891:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575688.086894:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393228288/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086900:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575688.086901:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575688.086904:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575688.086906:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575688.086908:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575688.086909:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575688.086911:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575688.086913:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:2.0:1788575688.086915:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575688.086916:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575688.086918:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:2.0:1788575688.086920:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575688.086921:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086922:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575688.086926:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393228416/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575713 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575688.086931:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575688.086932:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575688.086933:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575688.086934:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575688.086935:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575688.086936:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575688.086937:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575688.086938:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:2.0:1788575688.086939:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575688.086939:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575688.086940:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:2.0:1788575688.086941:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575688.086942:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086943:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.086945:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575688.086946:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575688.086947:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575688.086948:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575688.469522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575688.469531:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575688.469534:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575688.469537:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575688.469542:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575688.469544:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575688.469546:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575688.469548:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575688.469551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575688.469552:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575688.469556:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575688.469557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575688.469608:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575688.469612:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575688.469615:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575688.597486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575688.597493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575688.597495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575688.597500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575688.597503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575688.597507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.597510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575688.597513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575688.597515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575689.045504:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307292601, at 4307292608
00000100:00000010:0.0:1788575689.045518:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000100:00000040:1.0:1788575689.045600:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307292601, at 4307292608
00000100:00000010:1.0:1788575689.045605:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00010000:00000001:3.0:1788575689.493478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575689.493483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575689.493487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575689.493488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575689.493490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575689.493491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575689.493538:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575689.493541:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575689.493543:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575689.621495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575689.621502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575689.621504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575689.621509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575689.621513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575689.621516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575689.621519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575689.621521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575689.621524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575690.517522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575690.517529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575690.517535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575690.517536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575690.517540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575690.517541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575690.517601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575690.517606:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575690.517609:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575690.645507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575690.645514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575690.645516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575690.645520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575690.645522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575690.645525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575690.645527:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575690.645529:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575690.645531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575691.541511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575691.541516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575691.541520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575691.541521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575691.541523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575691.541524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575691.541576:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575691.541579:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575691.541583:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575691.669655:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575691.669663:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575691.669665:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575691.669677:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575691.669681:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575691.669685:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575691.669688:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575691.669690:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575691.669693:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575692.565511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575692.565517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575692.565522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575692.565523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575692.565526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575692.565527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575692.565575:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575692.565577:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575692.565580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575692.693510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575692.693515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575692.693517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575692.693522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575692.693525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575692.693528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575692.693529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575692.693531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575692.693533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575693.205477:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575693.205489:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575693.205492:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575693.205496:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575693.205501:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575693.205509:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12624
00000400:00000200:0.0:1788575693.205518:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575693.205524:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575693.205528:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575693.205531:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205534:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575693.205536:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205539:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575693.205543:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575693.205544:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575693.205547:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575693.205550:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575693.205553:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575693.205555:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575693.205558:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205560:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575693.205561:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205563:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575693.205567:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575693.205570:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205572:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575693.205574:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575693.205577:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575693.205580:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575693.205582:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575693.205584:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575693.205591:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575693.205593:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575693.205596:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575693.205597:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575693.205599:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575693.205601:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575693.205604:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575693.205610:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788575693.205615:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575693.205622:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575693.205640:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575693.205643:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575693.205646:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575693.205663:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575693.205666:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575693.205668:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575693.205669:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575693.205670:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575693.205672:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575693.205673:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12624
00000400:00000200:0.0:1788575693.205676:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575693.205678:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575693.205679:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575693.205681:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205682:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575693.205684:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205685:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575693.205686:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575693.205687:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575693.205688:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575693.205690:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575693.205691:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575693.205693:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575693.205695:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205696:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575693.205697:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205699:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575693.205701:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575693.205703:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205704:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575693.205705:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575693.205709:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575693.205709:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:0.0:1788575693.205713:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575693.205715:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575693.205715:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000008:00100000:0.0:1788575693.205718:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788575693.205719:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000008:00000001:0.0:1788575693.205721:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575693.205723:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575693.205726:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788575693.205729:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000040:2.0:1788575693.205730:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575693.205731:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575693.205732:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575693.205735:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575693.205737:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575693.205739:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575693.205743:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788575693.205748:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575693.205749:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575693.205753:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575693.205754:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575693.205758:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575693.205761:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575693.205763:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575693.205765:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575693.205769:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575693.205772:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575693.205774:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575693.205775:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575693.205779:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575693.205781:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575693.205783:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1788575693.205783:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393228544:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575693.205787:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575693.205792:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12624
00000100:00000001:2.0:1788575693.205792:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000400:00000200:0.0:1788575693.205798:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575693.205798:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575693.205802:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575693.205805:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:2.0:1788575693.205807:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:0.0:1788575693.205809:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575693.205813:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000010:2.0:1788575693.205815:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
00000100:00000001:0.0:1788575693.205816:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575693.205819:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575693.205821:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205824:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575693.205829:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1788575693.205829:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1788575693.205832:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575693.205836:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:2.0:1788575693.205836:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000020:00000040:0.0:1788575693.205839:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575693.205843:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:2.0:1788575693.205844:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000020:00000001:0.0:1788575693.205848:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:2.0:1788575693.205852:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393228544, portal 25
00000100:00000040:0.0:1788575693.205855:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575693.205855:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575693.205858:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000001:0.0:1788575693.205860:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575693.205863:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575693.205866:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575693.205867:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575693.205871:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575693.205877:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575693.205881:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788575693.205882:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575693.205885:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575693.205887:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575693.205889:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:2.0:1788575693.205890:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
02000000:00000001:0.0:1788575693.205892:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575693.205895:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:2.0:1788575693.205895:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393228544, offset 0
00000008:00100000:0.0:1788575693.205898:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575693.205901:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575693.205903:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575693.205904:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575693.205909:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788575693.205911:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575693.205914:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575693.205915:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575693.205917:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575693.205919:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575693.205920:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1788575693.205921:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575693.205925:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575693.205930:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000400:00000200:2.0:1788575693.205930:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575693.205935:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575693.205937:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00100000:0.0:1788575693.205941:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575693.205946:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575693.205947:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575693.205950:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575693.205957:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788575693.205958:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575693.205964:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575693.205964:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:0.0:1788575693.205971:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575693.205971:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331041:331041:256:4294967295] 192.168.202.110@tcp LPNI seq info [330998:330998:8:4294967295]
00000100:00000040:0.0:1788575693.205979:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12634)
00000400:00000200:2.0:1788575693.205979:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575693.205988:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575693.205993:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575693.205997:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575693.206003:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575693.206005:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575693.206009:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575693.206011:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575693.206014:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575693.206016:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575693.206020:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575693.206023:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575693.206027:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575693.206028:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575693.206032:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575693.206036:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206045:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575693.206049:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206056:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575693.206057:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575693.206063:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575693.206068:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575693.206070:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307297721
00000100:00000010:2.0:1788575693.206076:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c4e0.
00000100:00000001:2.0:1788575693.206082:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575693.206094:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307297721
00000100:00000010:2.0:1788575693.206098:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ccc0.
00000100:00000001:2.0:1788575693.206101:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575693.206105:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575693.206107:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575693.206110:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575693.206116:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575693/real 1788575693]  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575693.206126:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575693.206128:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575693.206130:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575693.206133:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575693.206135:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575693.206137:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575693.206141:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575693.206150:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206157:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575693.206158:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206161:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575693.206163:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575693.206167:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206175:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575693.206179:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206187:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575693.206190:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206191:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575693.206193:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206195:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575693.206197:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575693.206200:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575693.206204:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575693.206206:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575693.206211:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575693.206220:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393228544:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575693.206225:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206227:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575693.206230:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393228544/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206237:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575693.206239:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575693.206242:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575693.206245:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575693.206247:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575693.206249:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575693.206251:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575693.206253:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:2.0:1788575693.206257:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575693.206259:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575693.206261:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:2.0:1788575693.206264:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575693.206265:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206267:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206274:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575693.206276:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575693.206278:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575693.206279:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575693.206280:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575693.206284:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206290:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575693.206292:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575693.206294:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575693.206295:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575693.206297:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575693.206300:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228672:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575693.206303:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575693.206305:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575693.206306:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575693.206308:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575693.206310:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575693.206311:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575693.206314:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575693.206316:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575693.206318:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575693.206320:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393228672, portal 10
00000100:00000001:2.0:1788575693.206322:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575693.206322:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
00000100:00000040:2.0:1788575693.206327:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206331:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575693.206332:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575693.206334:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575693.206335:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393228672, offset 0
00000400:00000200:2.0:1788575693.206337:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575693.206340:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575693.206343:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575693.206345:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575693.206347:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575693.206357:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575693.206359:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575693.206365:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331042:331042:256:4294967295] 192.168.202.110@tcp LPNI seq info [330999:330999:8:4294967295]
00000400:00000200:2.0:1788575693.206369:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575693.206375:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575693.206380:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575693.206383:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575693.206387:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575693.206389:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575693.206418:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575693.206420:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575693.206424:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575693.206426:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575693.206429:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575693.206432:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575693.206436:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575693.206437:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575693.206440:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575693.206443:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206451:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575693.206454:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206461:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575693.206462:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575693.206465:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575693.206468:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575693.206472:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575693.206474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307296807
00000100:00000001:2.0:1788575693.206477:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307297721
00000100:00000001:2.0:1788575693.206479:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307296807
00000100:00000001:2.0:1788575693.206480:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307297721
00000100:00000001:2.0:1788575693.206482:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575693.206484:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575693.206486:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575693.206489:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575693/real 1788575693]  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575693.206497:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575693.206498:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575693.206500:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575693.206502:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575693.206504:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575693.206506:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575693.206509:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575693.206516:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206520:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575693.206521:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206522:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575693.206523:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575693.206525:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206529:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575693.206531:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206535:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575693.206536:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206537:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575693.206538:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206539:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575693.206541:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575693.206543:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575693.206544:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575693.206546:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575693.206548:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575693.206552:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228672:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575693.206555:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575693.206557:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206560:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575693.206561:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575693.206562:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575693.206563:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575693.206564:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575693.206566:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228800:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575693.206568:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575693.206569:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575693.206570:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575693.206571:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575693.206574:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575693.206575:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575693.206577:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575693.206579:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575693.206581:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575693.206583:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393228800, portal 4
00000100:00000001:2.0:1788575693.206584:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575693.206585:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:2.0:1788575693.206589:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206592:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575693.206594:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575693.206595:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575693.206597:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393228800, offset 0
00000400:00000200:2.0:1788575693.206599:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575693.206601:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575693.206604:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575693.206607:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575693.206609:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575693.206611:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575693.206613:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575693.206616:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331043:331043:256:4294967295] 192.168.202.110@tcp LPNI seq info [331000:331000:8:4294967295]
00000400:00000200:2.0:1788575693.206621:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575693.206637:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575693.206642:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575693.206646:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575693.206649:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575693.206652:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575693.206656:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575693.206658:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575693.206662:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575693.206664:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575693.206666:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575693.206669:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575693.206672:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575693.206674:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575693.206676:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575693.206680:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575718 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206686:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575693.206690:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575718 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206696:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575693.206697:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575693.206700:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575693.206704:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575693.206707:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575693.206709:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307296953
00000100:00000001:2.0:1788575693.206711:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307297721
00000100:00000001:2.0:1788575693.206713:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307296953
00000100:00000001:2.0:1788575693.206715:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307297721
00000100:00000001:2.0:1788575693.206717:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575693.206718:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575693.206720:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575693.206724:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575693/real 1788575693]  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575693.206731:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575693.206733:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575693.206734:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575693.206737:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575693.206738:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575693.206740:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575693.206744:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575693.206752:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206759:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575693.206760:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206761:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575693.206763:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575693.206766:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206772:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575693.206776:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206782:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575693.206784:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206785:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575693.206786:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206788:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575693.206789:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575693.206792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575693.206795:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575693.206797:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575693.206800:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575693.206808:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393228800:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575693.206812:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206814:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575693.206817:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393228672/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206823:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575693.206825:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575693.206827:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575693.206829:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575693.206831:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575693.206832:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575693.206834:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575693.206836:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:2.0:1788575693.206838:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575693.206839:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575693.206841:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:2.0:1788575693.206843:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575693.206845:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206846:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575693.206849:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393228800/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575718 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575693.206856:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575693.206857:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575693.206858:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575693.206860:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575693.206861:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575693.206862:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575693.206864:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575693.206865:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5c00.
02000000:00000001:2.0:1788575693.206867:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575693.206868:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575693.206869:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:2.0:1788575693.206871:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575693.206872:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206873:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.206876:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575693.206877:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575693.206879:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575693.206880:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575693.589497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575693.589501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575693.589504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575693.589505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575693.589507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575693.589508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575693.589563:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575693.589567:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575693.589571:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575693.717519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575693.717526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575693.717527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575693.717532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575693.717534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575693.717537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.717539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575693.717541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575693.717544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575694.165478:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307297721, at 4307297728
00000100:00000040:1.0:1788575694.165478:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307297721, at 4307297728
00000100:00000010:0.0:1788575694.165492:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00000100:00000010:1.0:1788575694.165495:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00010000:00000001:3.0:1788575694.613499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575694.613505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575694.613510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575694.613512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575694.613515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575694.613516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575694.613569:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575694.613573:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575694.613576:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575694.741476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575694.741484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575694.741487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575694.741493:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575694.741497:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575694.741501:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575694.741503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575694.741507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575694.741510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575695.637504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575695.637509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575695.637513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575695.637516:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575695.637518:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575695.637522:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575695.637525:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575695.637527:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575695.637529:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575695.637531:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575695.637535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575695.637536:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575695.637537:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575695.637539:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575695.637541:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575695.637542:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575695.637543:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575695.637545:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575695.637600:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575695.637604:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575695.637607:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575695.765523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575695.765529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575695.765531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575695.765535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575695.765537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575695.765539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575695.765541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575695.765543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575695.765545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575696.661501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575696.661507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575696.661512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575696.661513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575696.661516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575696.661518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575696.661572:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575696.661575:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575696.661579:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575696.789458:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575696.789465:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575696.789467:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575696.789472:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575696.789475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575696.789478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575696.789481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575696.789483:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575696.789486:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575697.685511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575697.685517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575697.685522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575697.685523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575697.685526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575697.685527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575697.685589:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575697.685593:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575697.685597:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575697.813745:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575697.813752:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575697.813755:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575697.813759:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575697.813763:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575697.813767:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575697.813770:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575697.813772:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575697.813775:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575698.325539:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575698.325549:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575698.325552:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575698.325555:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575698.325560:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575698.325567:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12629
00000400:00000200:3.0:1788575698.325576:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575698.325581:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575698.325584:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575698.325588:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325590:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575698.325593:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325595:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575698.325599:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575698.325602:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575698.325605:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575698.325608:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575698.325611:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575698.325613:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575698.325616:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325618:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575698.325620:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325622:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575698.325649:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575698.325654:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325657:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575698.325659:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575698.325663:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575698.325665:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575698.325668:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575698.325670:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575698.325679:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575698.325682:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575698.325686:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575698.325688:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575698.325691:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575698.325693:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575698.325696:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575698.325703:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575698.325710:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575698.325719:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575698.325729:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575698.325732:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575698.325737:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575698.325765:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575698.325769:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575698.325772:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575698.325774:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575698.325775:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575698.325778:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575698.325781:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12629
00000400:00000200:3.0:1788575698.325785:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575698.325788:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575698.325790:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575698.325793:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325795:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575698.325797:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325799:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575698.325801:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575698.325802:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575698.325804:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575698.325806:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575698.325809:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575698.325811:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575698.325813:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325815:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575698.325817:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325819:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575698.325822:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575698.325825:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325827:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575698.325829:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575698.325834:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575698.325837:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575698.325839:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575698.325843:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575698.325846:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575698.325848:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575698.325851:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788575698.325854:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575698.325857:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575698.325859:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575698.325861:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575698.325863:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575698.325864:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575698.325869:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:2.0:1788575698.325871:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575698.325873:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.325877:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00100000:3.0:1788575698.325880:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575698.325881:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575698.325891:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575698.325893:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575698.325897:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.325905:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1788575698.325906:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.325908:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575698.325913:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575698.325917:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575698.325921:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575698.325922:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575698.325926:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1788575698.325928:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393228928:192.168.202.110@tcp:250:kworker.0
00000100:00080000:3.0:1788575698.325933:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575698.325935:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575698.325937:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575698.325940:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:2.0:1788575698.325941:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575698.325945:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575698.325948:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575698.325950:0:186668:0:(import.c:520:import_select_connection()) Process entered
02000000:00000010:2.0:1788575698.325954:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
00000100:00080000:3.0:1788575698.325956:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12629
02000000:00000001:2.0:1788575698.325957:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575698.325965:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575698.325968:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575698.325972:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:2.0:1788575698.325974:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00080000:3.0:1788575698.325976:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575698.325982:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:2.0:1788575698.325984:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000001:3.0:1788575698.325985:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575698.325987:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.325992:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575698.325995:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000200:2.0:1788575698.325996:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393228928, portal 25
00000020:00000040:3.0:1788575698.325997:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575698.326001:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575698.326002:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000020:00000040:3.0:1788575698.326004:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:2.0:1788575698.326005:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000020:00000001:3.0:1788575698.326009:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575698.326014:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575698.326015:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575698.326021:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.326026:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575698.326028:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575698.326030:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.326033:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:2.0:1788575698.326035:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575698.326037:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:2.0:1788575698.326040:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00080000:3.0:1788575698.326043:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:2.0:1788575698.326046:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393228928, offset 0
00000100:00000001:3.0:1788575698.326047:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575698.326051:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575698.326054:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575698.326057:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575698.326058:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575698.326063:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575698.326065:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575698.326069:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:3.0:1788575698.326070:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575698.326074:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575698.326078:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000400:00000200:2.0:1788575698.326080:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:3.0:1788575698.326082:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575698.326086:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575698.326088:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1788575698.326089:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575698.326092:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575698.326094:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1788575698.326095:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575698.326099:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575698.326101:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000010:3.0:1788575698.326104:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000400:00000200:2.0:1788575698.326105:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575698.326111:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575698.326112:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331044:331044:256:4294967295] 192.168.202.110@tcp LPNI seq info [331001:331001:8:4294967295]
00000400:00000200:2.0:1788575698.326120:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1788575698.326120:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575698.326131:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575698.326136:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1788575698.326137:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575698.326146:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575698.326150:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1788575698.326153:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575698.326163:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575698.326165:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575698.326168:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:3.0:1788575698.326170:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575698.326176:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:2.1:1788575698.326179:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575698.326184:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12639)
00000800:00000200:2.1:1788575698.326184:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575698.326192:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575698.326194:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575698.326201:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575698.326206:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575698.326212:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575698.326222:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575698.326230:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575698.326235:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326245:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575698.326248:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326255:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.326257:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575698.326262:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575698.326269:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575698.326272:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307302841
00000100:00000010:2.0:1788575698.326277:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cae0.
00000100:00000001:2.0:1788575698.326283:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575698.326294:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307302841
00000100:00000010:2.0:1788575698.326298:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c660.
00000100:00000001:2.0:1788575698.326301:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575698.326303:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575698.326304:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.326306:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575698.326310:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575698/real 1788575698]  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575698.326317:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575698.326319:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575698.326321:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575698.326323:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575698.326324:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575698.326326:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575698.326328:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575698.326334:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326338:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575698.326339:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326340:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575698.326342:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575698.326344:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326348:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575698.326350:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326355:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575698.326357:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326358:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575698.326359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326361:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575698.326362:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575698.326364:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575698.326366:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575698.326368:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575698.326370:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575698.326375:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393228928:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575698.326378:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326379:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575698.326382:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393228928/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326388:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575698.326390:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575698.326438:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575698.326441:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575698.326443:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575698.326444:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575698.326445:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575698.326447:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575698.326449:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575698.326451:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575698.326452:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:2.0:1788575698.326454:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575698.326455:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326456:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326463:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575698.326465:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.326467:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575698.326469:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575698.326470:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575698.326475:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575698.326482:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575698.326485:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575698.326486:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575698.326488:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575698.326491:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229056:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575698.326494:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575698.326497:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575698.326498:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575698.326500:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575698.326503:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575698.326505:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575698.326508:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575698.326511:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575698.326513:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575698.326515:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393229056, portal 10
00000100:00000001:2.0:1788575698.326516:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575698.326517:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:2.0:1788575698.326523:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326530:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575698.326532:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575698.326535:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575698.326537:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393229056, offset 0
00000400:00000200:2.0:1788575698.326541:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575698.326551:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575698.326556:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575698.326559:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575698.326561:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575698.326563:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575698.326565:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575698.326567:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331045:331045:256:4294967295] 192.168.202.110@tcp LPNI seq info [331002:331002:8:4294967295]
00000400:00000200:2.0:1788575698.326569:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575698.326574:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575698.326577:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575698.326580:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575698.326584:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575698.326587:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575698.326591:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575698.326593:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575698.326597:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575698.326599:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575698.326602:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575698.326605:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575698.326608:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575698.326609:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575698.326612:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575698.326616:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326624:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575698.326638:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326645:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.326647:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575698.326650:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575698.326653:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575698.326656:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575698.326658:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307301927
00000100:00000001:2.0:1788575698.326661:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307302841
00000100:00000001:2.0:1788575698.326663:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307301927
00000100:00000001:2.0:1788575698.326665:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307302841
00000100:00000001:2.0:1788575698.326667:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575698.326669:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.326671:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575698.326674:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575698/real 1788575698]  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575698.326682:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575698.326683:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575698.326685:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575698.326687:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575698.326688:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575698.326690:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575698.326693:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575698.326701:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326707:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575698.326709:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326711:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575698.326712:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575698.326715:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326721:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575698.326725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326731:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575698.326733:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326735:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575698.326736:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326738:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575698.326740:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575698.326743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575698.326746:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575698.326748:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575698.326752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575698.326771:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229056:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575698.326776:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575698.326779:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326785:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575698.326787:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575698.326789:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575698.326791:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575698.326793:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575698.326795:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575698.326799:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575698.326800:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575698.326801:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575698.326802:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575698.326805:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575698.326806:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575698.326809:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575698.326810:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575698.326813:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575698.326815:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393229184, portal 4
00000100:00000001:2.0:1788575698.326816:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575698.326817:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:2.0:1788575698.326821:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326828:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575698.326830:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575698.326831:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575698.326832:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393229184, offset 0
00000400:00000200:2.0:1788575698.326835:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575698.326837:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575698.326839:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575698.326841:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575698.326843:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575698.326844:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575698.326846:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575698.326848:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331046:331046:256:4294967295] 192.168.202.110@tcp LPNI seq info [331003:331003:8:4294967295]
00000400:00000200:2.0:1788575698.326850:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575698.326855:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575698.326857:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575698.326859:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575698.326862:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575698.326864:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575698.326866:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575698.326868:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575698.326870:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575698.326871:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575698.326873:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575698.326875:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575698.326877:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575698.326878:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575698.326879:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575698.326881:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326886:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575698.326888:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575723 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326895:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.326896:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575698.326898:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575698.326899:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575698.326901:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575698.326903:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307302073
00000100:00000001:2.0:1788575698.326905:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307302841
00000100:00000001:2.0:1788575698.326907:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307302073
00000100:00000001:2.0:1788575698.326909:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307302841
00000100:00000001:2.0:1788575698.326911:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575698.326913:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.326915:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575698.326918:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575698/real 1788575698]  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575698.326925:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575698.326927:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575698.326929:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575698.326931:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575698.326933:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575698.326935:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575698.326938:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575698.326945:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326951:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575698.326953:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326954:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575698.326956:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575698.326959:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326966:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575698.326970:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.326976:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575698.326979:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326980:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575698.326981:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.326983:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575698.326984:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575698.326987:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575698.326990:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575698.326992:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575698.326996:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575698.327004:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575698.327007:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.327009:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575698.327012:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393229056/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.327016:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575698.327017:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575698.327018:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575698.327020:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575698.327021:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575698.327022:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575698.327023:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575698.327024:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575698.327026:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575698.327027:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575698.327028:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788575698.327029:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575698.327030:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.327031:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575698.327033:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393229184/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575723 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575698.327037:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575698.327038:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575698.327038:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575698.327039:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575698.327040:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575698.327041:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575698.327042:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575698.327042:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575698.327043:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575698.327044:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575698.327045:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:2.0:1788575698.327046:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575698.327046:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.327047:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.327050:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575698.327051:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575698.327052:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575698.327053:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575698.709488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575698.709495:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575698.709498:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575698.709501:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575698.709505:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575698.709507:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575698.709509:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575698.709511:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575698.709514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575698.709516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575698.709519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575698.709520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575698.709570:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575698.709573:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575698.709575:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575698.837528:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575698.837536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575698.837538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575698.837543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575698.837546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575698.837550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.837553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575698.837555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575698.837558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575699.285504:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307302841, at 4307302848
00000100:00000040:1.0:1788575699.285504:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307302841, at 4307302848
00000100:00000010:0.0:1788575699.285520:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00000100:00000010:1.0:1788575699.285521:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00010000:00000001:3.0:1788575699.733511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575699.733524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575699.733533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575699.733534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575699.733537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575699.733538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575699.733595:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575699.733600:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575699.733603:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575699.861497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575699.861502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575699.861504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575699.861508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575699.861511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575699.861513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575699.861515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575699.861517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575699.861519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575700.757551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575700.757561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575700.757568:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575700.757569:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575700.757574:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575700.757575:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575700.757648:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575700.757654:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575700.757659:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575700.885509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575700.885516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575700.885518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575700.885523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575700.885526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575700.885528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575700.885531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575700.885532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575700.885535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575701.781511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575701.781517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575701.781523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575701.781525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575701.781528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575701.781530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575701.781593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575701.781597:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575701.781600:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575701.909485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575701.909493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575701.909495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575701.909500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575701.909504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575701.909508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575701.909510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575701.909513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575701.909515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575702.805504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575702.805509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575702.805514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575702.805515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575702.805517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575702.805518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575702.805589:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575702.805591:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575702.805593:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575702.933509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575702.933515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575702.933518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575702.933522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575702.933526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575702.933529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575702.933531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575702.933533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575702.933536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575703.445476:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575703.445486:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575703.445489:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575703.445493:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575703.445497:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575703.445505:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12634
00000400:00000200:3.0:1788575703.445515:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575703.445522:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575703.445526:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575703.445530:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445533:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575703.445536:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445539:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575703.445542:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575703.445544:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575703.445547:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575703.445550:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575703.445553:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575703.445555:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575703.445559:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445561:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575703.445563:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445565:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575703.445568:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575703.445571:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445573:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575703.445575:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575703.445577:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575703.445579:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575703.445580:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575703.445582:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575703.445589:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000020:00000040:3.0:1788575703.445592:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575703.445595:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575703.445596:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575703.445598:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575703.445600:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575703.445603:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575703.445609:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575703.445613:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575703.445620:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575703.445642:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575703.445645:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575703.445648:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575703.445664:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575703.445666:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575703.445669:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575703.445670:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575703.445672:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575703.445675:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575703.445678:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12634
00000400:00000200:3.0:1788575703.445682:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575703.445684:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575703.445687:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575703.445690:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445692:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575703.445694:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445697:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575703.445699:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575703.445701:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1788575703.445701:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575703.445704:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575703.445707:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1788575703.445708:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575703.445716:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575703.445717:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575703.445720:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575703.445725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575703.445725:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445729:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575703.445733:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445737:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575703.445741:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575703.445743:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575703.445746:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575703.445746:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445751:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575703.445752:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575703.445753:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575703.445755:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575703.445758:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575703.445759:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575703.445762:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575703.445763:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:2.0:1788575703.445768:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393229312:192.168.202.110@tcp:250:kworker.0
00000008:00100000:3.0:1788575703.445768:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575703.445773:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.445775:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575703.445776:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575703.445779:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575703.445781:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575703.445781:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1180.
02000000:00000001:2.0:1788575703.445785:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1788575703.445786:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575703.445789:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575703.445790:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575703.445791:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
00000100:00000001:3.0:1788575703.445792:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575703.445795:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575703.445795:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575703.445798:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575703.445802:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1788575703.445803:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000400:00000010:2.0:1788575703.445806:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00000001:3.0:1788575703.445807:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575703.445812:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00100000:3.0:1788575703.445813:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788575703.445818:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393229312, portal 25
00000100:00000001:3.0:1788575703.445820:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575703.445822:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575703.445823:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.445824:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:3.0:1788575703.445829:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca1dca9a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575703.445832:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575703.445845:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.445846:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1788575703.445848:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575703.445852:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575703.445854:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:2.0:1788575703.445855:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000001:3.0:1788575703.445856:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575703.445858:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:2.0:1788575703.445859:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393229312, offset 0
00000100:00000001:3.0:1788575703.445862:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575703.445866:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12634
00000400:00000200:3.0:1788575703.445872:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575703.445874:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575703.445875:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575703.445876:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575703.445879:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445881:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575703.445882:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445884:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575703.445885:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1788575703.445887:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1788575703.445887:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575703.445890:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575703.445892:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575703.445895:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575703.445896:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575703.445900:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575703.445903:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:3.0:1788575703.445906:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575703.445910:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1788575703.445914:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575703.445916:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445919:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575703.445922:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575703.445926:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575703.445930:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575703.445931:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331047:331047:256:4294967295] 192.168.202.110@tcp LPNI seq info [331004:331004:8:4294967295]
00000100:00080000:3.0:1788575703.445937:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575703.445942:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575703.445943:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575703.445946:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575703.445949:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575703.445951:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575703.445954:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575703.445957:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575703.445959:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000008:00000001:3.0:1788575703.445960:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575703.445962:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575703.445967:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000020:00000040:3.0:1788575703.445971:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.0:1788575703.445972:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575703.445974:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575703.445976:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575703.445978:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575703.445979:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:2.0:1788575703.445980:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
02000000:00000001:3.0:1788575703.445981:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575703.445988:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000800:00000200:2.0:1788575703.445991:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575703.445992:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575703.445997:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00100000:3.0:1788575703.445999:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1788575703.446008:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575703.446010:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575703.446013:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575703.446014:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:3.0:1788575703.446018:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:2.0:1788575703.446024:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575703.446029:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000100:00000001:3.0:1788575703.446029:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575703.446033:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575703.446035:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1788575703.446036:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575703.446041:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1788575703.446042:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12644)
00000400:00000200:2.0:1788575703.446046:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575703.446049:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575703.446053:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575703.446056:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446063:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575703.446065:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446070:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.446071:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575703.446075:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575703.446078:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575703.446080:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307307961
00000100:00000010:2.0:1788575703.446084:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ce40.
00000100:00000001:2.0:1788575703.446088:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575703.446098:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307307961
00000100:00000010:2.0:1788575703.446100:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c240.
00000100:00000001:2.0:1788575703.446103:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575703.446105:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575703.446107:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.446108:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575703.446111:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575703/real 1788575703]  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575703.446118:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575703.446119:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575703.446121:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575703.446123:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575703.446124:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575703.446127:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575703.446129:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575703.446135:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446138:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575703.446139:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446141:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575703.446142:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575703.446145:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446148:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575703.446151:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446155:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575703.446157:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446158:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575703.446159:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446160:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575703.446162:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575703.446164:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575703.446166:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575703.446168:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575703.446170:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575703.446175:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393229312:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575703.446178:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446179:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575703.446181:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393229312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446184:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575703.446186:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575703.446188:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575703.446190:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575703.446191:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575703.446192:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575703.446193:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575703.446195:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575703.446197:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575703.446198:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575703.446199:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:2.0:1788575703.446202:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575703.446203:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446204:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446209:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575703.446211:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.446213:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575703.446214:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575703.446215:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575703.446219:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446230:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575703.446232:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575703.446233:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575703.446235:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575703.446237:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575703.446240:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229440:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575703.446243:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575703.446245:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575703.446246:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575703.446248:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575703.446251:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575703.446253:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575703.446256:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575703.446259:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575703.446262:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575703.446265:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393229440, portal 10
00000100:00000001:2.0:1788575703.446267:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575703.446268:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000100:00000040:2.0:1788575703.446274:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446280:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575703.446282:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575703.446283:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575703.446285:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393229440, offset 0
00000400:00000200:2.0:1788575703.446287:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575703.446290:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575703.446292:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575703.446294:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575703.446296:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575703.446297:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575703.446299:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575703.446301:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331048:331048:256:4294967295] 192.168.202.110@tcp LPNI seq info [331005:331005:8:4294967295]
00000400:00000200:2.0:1788575703.446304:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575703.446308:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575703.446310:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575703.446313:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575703.446315:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575703.446316:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575703.446319:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575703.446321:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575703.446325:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575703.446327:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575703.446329:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575703.446332:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575703.446335:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575703.446338:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575703.446340:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575703.446343:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446351:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575703.446354:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446360:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.446362:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575703.446365:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575703.446367:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575703.446369:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575703.446370:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307307046
00000100:00000001:2.0:1788575703.446372:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307307961
00000100:00000001:2.0:1788575703.446374:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307307046
00000100:00000001:2.0:1788575703.446375:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307307961
00000100:00000001:2.0:1788575703.446376:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575703.446379:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.446390:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575703.446417:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575703/real 1788575703]  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575703.446426:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575703.446427:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575703.446428:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575703.446430:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575703.446431:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575703.446433:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575703.446435:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575703.446440:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446444:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575703.446445:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446446:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575703.446446:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575703.446448:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446452:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575703.446454:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446458:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575703.446460:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446460:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575703.446461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446462:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575703.446464:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575703.446465:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575703.446469:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575703.446471:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575703.446474:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575703.446506:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229440:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575703.446510:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575703.446512:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446516:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575703.446517:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575703.446519:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575703.446520:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575703.446521:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575703.446523:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229568:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575703.446525:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575703.446526:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575703.446527:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575703.446528:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575703.446531:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575703.446532:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575703.446534:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575703.446536:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575703.446538:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575703.446541:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393229568, portal 4
00000100:00000001:2.0:1788575703.446542:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575703.446542:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:2.0:1788575703.446546:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446550:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575703.446552:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575703.446553:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575703.446555:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393229568, offset 0
00000400:00000200:2.0:1788575703.446557:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575703.446560:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575703.446562:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575703.446564:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575703.446566:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575703.446567:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575703.446569:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575703.446570:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331049:331049:256:4294967295] 192.168.202.110@tcp LPNI seq info [331006:331006:8:4294967295]
00000400:00000200:2.0:1788575703.446573:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575703.446577:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575703.446580:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575703.446582:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575703.446584:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575703.446586:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575703.446588:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575703.446589:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575703.446592:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575703.446593:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:2.0:1788575703.446595:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575703.446597:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575703.446599:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:2.0:1788575703.446600:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:2.0:1788575703.446602:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575703.446604:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446609:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575703.446610:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446614:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.446615:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575703.446618:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575703.446620:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575703.446623:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575703.446636:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307307193
00000100:00000001:2.0:1788575703.446638:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307307961
00000100:00000001:2.0:1788575703.446640:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307307193
00000100:00000001:2.0:1788575703.446642:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307307961
00000100:00000001:2.0:1788575703.446644:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575703.446646:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.446648:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575703.446653:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575703/real 1788575703]  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575703.446660:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575703.446662:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575703.446664:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575703.446666:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575703.446668:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575703.446670:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575703.446673:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575703.446681:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446686:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575703.446688:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446689:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575703.446691:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575703.446694:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446701:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575703.446705:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446711:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575703.446714:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446716:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575703.446717:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446718:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575703.446719:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575703.446730:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575703.446733:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575703.446734:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575703.446737:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575703.446742:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229568:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575703.446744:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446745:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575703.446747:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393229440/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446751:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575703.446752:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575703.446753:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575703.446755:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575703.446756:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575703.446757:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575703.446758:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575703.446759:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575703.446761:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575703.446762:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575703.446763:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:2.0:1788575703.446764:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575703.446765:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446766:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575703.446768:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393229568/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575703.446772:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575703.446773:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575703.446773:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575703.446774:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575703.446775:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575703.446776:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575703.446776:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575703.446777:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575703.446778:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575703.446779:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575703.446779:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:2.0:1788575703.446781:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575703.446781:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446782:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.446784:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575703.446785:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575703.446786:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575703.446787:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575703.829499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575703.829504:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575703.829508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575703.829508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575703.829511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575703.829511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575703.829577:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575703.829581:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575703.829584:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575703.957507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575703.957516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575703.957517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575703.957523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575703.957526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575703.957530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.957532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575703.957534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575703.957537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575704.405502:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307307961, at 4307307968
00000100:00000010:1.0:1788575704.405515:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000100:00000040:0.0:1788575704.405598:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307307961, at 4307307968
00000100:00000010:0.0:1788575704.405604:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00010000:00000001:3.0:1788575704.853504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575704.853510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575704.853523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575704.853530:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575704.853532:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575704.853535:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575704.853538:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575704.853540:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575704.853541:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575704.853543:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575704.853546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575704.853548:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575704.853549:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575704.853550:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575704.853551:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575704.853552:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575704.853554:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575704.853555:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575704.853615:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575704.853619:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575704.853622:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575704.981514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575704.981521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575704.981523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575704.981529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575704.981534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575704.981537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575704.981540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575704.981541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575704.981543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575705.877534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575705.877541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575705.877546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575705.877547:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575705.877551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575705.877553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575705.877621:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575705.877636:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575705.877640:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575706.005513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575706.005520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575706.005522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575706.005527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575706.005530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575706.005534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575706.005537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575706.005539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575706.005542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575706.901763:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575706.901770:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575706.901776:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575706.901777:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575706.901781:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575706.901782:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575706.901922:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575706.901925:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575706.901928:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575707.029519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575707.029524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575707.029526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575707.029530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575707.029533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575707.029536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575707.029538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575707.029540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575707.029542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575707.925508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575707.925517:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575707.925519:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575707.925522:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575707.925527:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575707.925529:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575707.925531:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575707.925534:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575707.925538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575707.925539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575707.925542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575707.925543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575707.925676:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575707.925680:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575707.925684:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575708.053549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575708.053557:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575708.053559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575708.053564:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575708.053568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575708.053572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575708.053574:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575708.053577:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575708.053580:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575708.565468:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575708.565477:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575708.565480:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575708.565484:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575708.565488:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575708.565494:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12639
00000400:00000200:3.0:1788575708.565502:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575708.565507:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575708.565509:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575708.565512:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565514:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575708.565516:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565517:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575708.565520:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575708.565521:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575708.565524:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575708.565527:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575708.565528:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575708.565530:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575708.565532:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565533:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575708.565534:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565535:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575708.565537:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575708.565539:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565541:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575708.565543:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575708.565545:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575708.565547:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575708.565549:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575708.565550:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575708.565557:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788575708.565560:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575708.565562:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575708.565563:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575708.565566:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575708.565568:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575708.565572:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575708.565579:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa000.
00000100:00000001:3.0:1788575708.565585:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575708.565593:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575708.565603:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575708.565607:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575708.565612:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575708.565643:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575708.565646:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575708.565649:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575708.565651:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575708.565653:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575708.565655:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575708.565658:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12639
00000400:00000200:3.0:1788575708.565662:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575708.565665:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575708.565668:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575708.565670:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565673:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575708.565675:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565677:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575708.565679:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575708.565680:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575708.565682:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575708.565684:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575708.565685:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575708.565686:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575708.565688:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565690:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575708.565691:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565692:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575708.565693:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575708.565694:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565696:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575708.565697:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575708.565701:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575708.565702:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575708.565703:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575708.565707:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575708.565709:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575708.565710:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575708.565713:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575708.565714:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575708.565716:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575708.565716:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575708.565718:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575708.565718:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575708.565720:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575708.565722:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
00000100:00000001:3.0:1788575708.565725:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575708.565728:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575708.565733:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575708.565734:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575708.565737:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575708.565742:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575708.565743:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575708.565745:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575708.565746:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575708.565747:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575708.565749:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575708.565750:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12639
00000400:00000200:3.0:1788575708.565752:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575708.565754:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575708.565755:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575708.565757:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565758:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575708.565759:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565761:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575708.565762:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575708.565763:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575708.565764:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575708.565765:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575708.565766:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575708.565768:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575708.565769:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565770:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575708.565771:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565772:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575708.565773:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575708.565774:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575708.565776:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575708.565776:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575708.565778:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575708.565779:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575708.565780:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575708.565781:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575708.565782:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575708.565783:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575708.565785:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000020:00000040:3.0:1788575708.565786:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575708.565787:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575708.565788:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575708.565789:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575708.565789:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575708.565790:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575708.565793:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788575708.565795:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575708.565797:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575708.565805:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575708.565806:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575708.565808:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575708.565812:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575708.565813:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575708.565815:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575708.565818:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12649)
00000100:00000001:2.0:1788575708.565886:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575708.565890:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575708.565892:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575708.565897:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575708.565902:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575708.565909:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575708.565912:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575708.565914:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575708.565915:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575708.565918:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393229696:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575708.565920:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575708.565924:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575708.565925:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575708.565927:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575708.565931:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575708.565932:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575708.565936:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575708.565939:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575708.565944:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575708.565948:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393229696, portal 25
00000100:00000001:2.0:1788575708.565950:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575708.565951:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:2.0:1788575708.565955:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575708.565960:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575708.565962:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575708.565963:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65220.
00000100:00000200:2.0:1788575708.565966:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393229696, offset 0
00000400:00000200:2.0:1788575708.565969:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575708.565975:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575708.565985:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575708.565989:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575708.565992:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575708.565995:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575708.565997:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575708.566000:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331050:331050:256:4294967295] 192.168.202.110@tcp LPNI seq info [331007:331007:8:4294967295]
00000400:00000200:2.0:1788575708.566003:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575708.566010:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575708.566014:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575708.566017:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575708.566022:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575708.566025:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575708.566036:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575708.566040:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575708.566043:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307313081
00000100:00000010:2.0:1788575708.566047:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd20.
00000100:00000001:2.0:1788575708.566054:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575708.566068:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307313081
00000100:00000010:2.0:1788575708.566071:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c360.
00000100:00000001:2.0:1788575708.566074:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575708.566077:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575708.566079:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575708.566082:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575708.566085:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575708.566092:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575708.566093:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575708.566095:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575708.566097:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575708.566099:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575708.566101:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229824:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575708.566104:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575708.566106:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575708.566107:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575708.566108:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575708.566111:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575708.566112:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575708.566116:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575708.566118:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575708.566121:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575708.566123:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393229824, portal 10
00000100:00000001:2.0:1788575708.566125:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575708.566126:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:2.0:1788575708.566132:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575708.566137:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575708.566139:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575708.566141:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65550.
00000100:00000200:2.0:1788575708.566143:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393229824, offset 0
00000400:00000200:2.0:1788575708.566146:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575708.566149:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575708.566152:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575708.566155:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575708.566157:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575708.566159:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575708.566161:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575708.566164:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331051:331051:255:4294967295] 192.168.202.110@tcp LPNI seq info [331008:331008:7:4294967295]
00000400:00000200:2.0:1788575708.566168:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575708.566173:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575708.566177:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575708.566180:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133b00.
00000800:00000200:2.0:1788575708.566184:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575708.566187:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575708.566190:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575708.566193:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575708.566196:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575708.566198:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307312166
00000100:00000001:2.0:1788575708.566200:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307313081
00000100:00000001:2.0:1788575708.566202:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307312166
00000100:00000001:2.0:1788575708.566205:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307313081
00000100:00000001:2.0:1788575708.566207:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575708.566209:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575708.566211:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575708.566216:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575708.566222:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575708.566223:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575708.566233:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575708.566236:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575708.566238:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575708.566241:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229952:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575708.566244:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575708.566246:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575708.566247:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575708.566249:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575708.566252:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575708.566254:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575708.566257:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575708.566260:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575708.566263:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575708.566266:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393229952, portal 4
00000100:00000001:2.0:1788575708.566268:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575708.566269:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:2.0:1788575708.566275:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575708.566280:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575708.566282:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575708.566284:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65770.
00000100:00000200:2.0:1788575708.566287:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393229952, offset 0
00000400:00000200:2.0:1788575708.566291:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575708.566294:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575708.566299:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575708.566302:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575708.566305:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575708.566307:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575708.566309:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575708.566312:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331052:331052:254:4294967295] 192.168.202.110@tcp LPNI seq info [331009:331009:6:4294967295]
00000400:00000200:2.0:1788575708.566315:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575708.566321:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575708.566325:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575708.566328:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575708.566332:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575708.566335:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575708.566338:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575708.566341:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575708.566344:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575708.566346:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307312312
00000100:00000001:2.0:1788575708.566348:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307313081
00000100:00000001:2.0:1788575708.566350:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307312312
00000100:00000001:2.0:1788575708.566352:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307313081
00000100:00000001:2.0:1788575708.566354:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575708.566356:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575708.566358:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575708.566361:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575708.566367:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575708.566370:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575708.566372:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575708.566373:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575708.566374:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575708.566376:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575708.949508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575708.949515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575708.949521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575708.949522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575708.949526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575708.949527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575708.949619:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575708.949649:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575708.949652:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575709.077520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575709.077528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575709.077530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575709.077536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575709.077540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575709.077543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575709.077546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575709.077548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575709.077551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575709.525513:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307313081, at 4307313088
00000100:00000010:0.0:1788575709.525526:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00000100:00000040:1.0:1788575709.525605:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307313081, at 4307313088
00000100:00000010:1.0:1788575709.525612:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00010000:00000001:3.0:1788575709.973579:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575709.973586:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575709.973591:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575709.973592:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575709.973596:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575709.973597:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575709.973657:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575709.973661:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575709.973664:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575710.101694:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575710.101701:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575710.101704:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575710.101709:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575710.101712:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575710.101716:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575710.101719:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575710.101721:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575710.101725:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575710.997470:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575710.997474:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575710.997478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575710.997479:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575710.997481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575710.997482:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575710.997527:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575710.997531:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575710.997534:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575711.125510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575711.125517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575711.125519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575711.125524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575711.125526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575711.125529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.125531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575711.125533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.125534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788575711.637683:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575711.637689:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788575711.637693:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000400:00000100:3.0:1788575711.637700:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:3.0:1788575711.637736:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:3.0:1788575711.637745:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575711.637750:0:2351:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:3.0:1788575711.637761:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575711.637765:0:2351:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:3.0:1788575711.637769:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1788575711.637772:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:3.0:1788575711.637776:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1788575711.637779:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1788575711.637784:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65220
00000400:00000010:3.0:1788575711.637787:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65220.
00000100:00000001:3.0:1788575711.637792:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575711.637801:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575711.637830:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575711.637835:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1788575711.637839:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000001:3.0:1788575711.637846:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000800:00000001:0.0:1788575711.637847:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000001:3.0:1788575711.637849:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000001:0.0:1788575711.637861:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000100:3.0:1788575711.637861:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1788575711.637867:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133b00.
00000800:00000001:0.0:1788575711.637868:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000400:00000200:3.0:1788575711.637871:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000800:00000001:0.0:1788575711.637873:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1788575711.637878:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1788575711.637882:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65550
00000400:00000010:3.0:1788575711.637884:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65550.
00000100:00000001:3.0:1788575711.637892:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575711.637897:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575711.637905:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575711.637908:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575711.637915:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575711.637928:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:3.0:1788575711.637933:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1788575711.637935:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:3.0:1788575711.637938:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1788575711.637942:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1788575711.637945:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65770
00000400:00000010:3.0:1788575711.637947:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65770.
00000100:00000001:3.0:1788575711.637949:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575711.637953:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575711.637959:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575711.637963:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575711.637968:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575711.637969:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575711.638353:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575711.638357:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575711.638359:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575711.638366:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575708/real 1788575711]  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575711.638378:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575711.638381:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575711.638383:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575711.638387:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575711.638388:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575711.638419:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575711.638423:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575711.638431:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638437:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575711.638439:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638442:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575711.638444:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575711.638450:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638457:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575711.638462:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638473:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575711.638476:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638477:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575711.638478:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638481:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575711.638483:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575711.638488:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575711.638493:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575711.638497:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575711.638502:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575711.638509:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393229696:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575711.638515:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575711.638518:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575708/real 1788575711]  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575711.638532:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575711.638533:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575711.638535:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575711.638539:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575711.638540:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575711.638542:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575711.638546:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575711.638554:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638560:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575711.638561:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638563:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575711.638564:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575711.638568:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638574:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575711.638578:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638585:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575711.638587:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638589:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575711.638590:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638591:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575711.638594:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575711.638596:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575711.638600:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575711.638603:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575711.638607:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575711.638614:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229824:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575711.638618:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575711.638637:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575708/real 1788575711]  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575711.638646:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575711.638647:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575711.638650:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575711.638652:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575711.638653:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575711.638656:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575711.638659:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575711.638666:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638673:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575711.638674:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638676:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575711.638678:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575711.638681:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638686:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575711.638689:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638695:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575711.638697:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638700:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575711.638701:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638703:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575711.638705:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575711.638707:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575711.638711:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575711.638713:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575711.638718:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575711.638725:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393229952:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575711.638729:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638731:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575711.638735:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393229696/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638741:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575711.638745:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575711.638750:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575711.638754:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575711.638758:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575711.638760:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575711.638763:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575711.638766:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:2.0:1788575711.638771:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575711.638774:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575711.638776:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788575711.638780:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575711.638781:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638782:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575711.638785:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393229824/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638791:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575711.638793:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575711.638794:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575711.638796:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575711.638798:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575711.638799:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575711.638801:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575711.638802:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:2.0:1788575711.638805:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575711.638806:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575711.638807:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:2.0:1788575711.638810:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575711.638811:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638812:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575711.638815:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393229952/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575711.638820:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575711.638822:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575711.638823:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575711.638826:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575711.638827:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575711.638828:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575711.638830:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575711.638831:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575711.638834:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575711.638835:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575711.638836:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788575711.638838:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575711.638840:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638842:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575711.638852:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575711.638855:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575711.638857:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575711.638859:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575712.021496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575712.021502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575712.021507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575712.021508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575712.021511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575712.021512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575712.021560:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575712.021563:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575712.021565:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575712.149503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575712.149511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575712.149514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575712.149519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575712.149524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575712.149528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575712.149531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575712.149534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575712.149537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575713.045514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575713.045520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575713.045524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575713.045525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575713.045528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575713.045529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575713.045585:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575713.045587:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575713.045590:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575713.173513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575713.173521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575713.173523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575713.173528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575713.173531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575713.173535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.173538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575713.173541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.173544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575713.685601:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575713.685609:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575713.685611:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575713.685613:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575713.685617:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575713.685640:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12644
00000400:00000200:3.0:1788575713.685647:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575713.685652:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575713.685655:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575713.685657:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685660:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575713.685662:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685664:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575713.685668:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575713.685670:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575713.685673:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575713.685676:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575713.685678:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575713.685680:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575713.685683:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685685:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575713.685686:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685688:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575713.685692:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575713.685695:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685698:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575713.685700:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575713.685703:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575713.685706:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575713.685708:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575713.685711:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575713.685717:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000020:00000040:3.0:1788575713.685721:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575713.685725:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575713.685726:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575713.685729:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575713.685731:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575713.685735:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575713.685742:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575713.685748:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575713.685758:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575713.685775:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575713.685779:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575713.685784:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575713.685811:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575713.685815:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575713.685818:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575713.685820:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575713.685822:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575713.685825:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575713.685827:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12644
00000400:00000200:3.0:1788575713.685831:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575713.685834:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575713.685837:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575713.685839:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685841:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575713.685843:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685845:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575713.685847:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575713.685848:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575713.685850:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575713.685851:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575713.685853:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575713.685855:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575713.685858:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685859:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575713.685861:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685863:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575713.685867:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575713.685869:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685872:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575713.685874:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575713.685879:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575713.685881:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575713.685883:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575713.685888:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575713.685891:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575713.685893:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575713.685896:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575713.685900:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575713.685902:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575713.685904:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575713.685905:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575713.685906:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575713.685908:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575713.685912:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575713.685915:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575713.685920:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575713.685929:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575713.685931:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575713.685934:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575713.685941:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575713.685944:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575713.685947:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575713.685949:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575713.685950:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575713.685953:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575713.685956:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12644
00000400:00000200:3.0:1788575713.685960:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575713.685963:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575713.685964:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575713.685967:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685969:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575713.685970:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685972:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575713.685974:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575713.685975:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575713.685976:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575713.685978:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575713.685980:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575713.685981:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575713.685984:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685985:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575713.685987:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685988:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575713.685990:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575713.685992:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575713.685995:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575713.685997:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575713.685999:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575713.686001:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575713.686003:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575713.686005:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575713.686007:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575713.686008:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575713.686011:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788575713.686013:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575713.686016:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575713.686018:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575713.686019:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575713.686020:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575713.686022:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575713.686025:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575713.686028:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575713.686032:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575713.686038:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575713.686040:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575713.686043:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575713.686050:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575713.686052:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575713.686055:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575713.686060:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12654)
00000100:00000001:2.0:1788575713.686136:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575713.686140:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575713.686142:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575713.686148:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.686156:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575713.686158:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575713.686161:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575713.686163:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575713.686166:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575713.686169:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393230080:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575713.686173:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575713.686177:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575713.686179:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575713.686182:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575713.686186:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575713.686188:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575713.686193:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575713.686197:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575713.686204:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575713.686211:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393230080, portal 25
00000100:00000001:2.0:1788575713.686213:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575713.686215:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:2.0:1788575713.686220:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.686227:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575713.686229:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575713.686232:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575713.686235:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393230080, offset 0
00000400:00000200:2.0:1788575713.686242:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575713.686250:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575713.686256:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575713.686262:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575713.686265:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575713.686268:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575713.686271:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575713.686275:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331053:331053:256:4294967295] 192.168.202.110@tcp LPNI seq info [331010:331010:8:4294967295]
00000400:00000200:2.0:1788575713.686279:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575713.686288:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575713.686295:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575713.686300:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575713.686307:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575713.686311:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575713.686317:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575713.686320:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575713.686324:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575713.686327:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575713.686333:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575713.686337:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575713.686342:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575713.686345:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575713.686348:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575713.686352:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.686361:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575713.686364:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.686372:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575713.686374:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575713.686380:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575713.686384:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575713.686387:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307318201
00000100:00000010:2.0:1788575713.686427:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c8a0.
00000100:00000001:2.0:1788575713.686434:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575713.686453:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307318202
00000100:00000010:2.0:1788575713.686457:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c960.
00000100:00000001:2.0:1788575713.686461:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575713.686463:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575713.686466:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575713.686469:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575713.686473:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575713/real 1788575713]  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575713.686483:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575713.686485:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575713.686487:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575713.686490:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575713.686491:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575713.686494:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575713.686497:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575713.686504:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.686512:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575713.686514:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.686516:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575713.686519:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575713.686523:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.686530:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575713.686536:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.686543:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575713.686546:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.686547:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575713.686548:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.686551:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575713.686553:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575713.686556:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575713.686559:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575713.686562:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575713.686566:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575713.686574:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393230080:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575713.686578:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575713.686581:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.686588:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575713.686589:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575713.686591:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575713.686593:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575713.686594:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575713.686597:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575713.686600:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575713.686602:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575713.686603:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575713.686605:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575713.686608:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575713.686610:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575713.686614:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575713.686616:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575713.686620:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575713.686634:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393230208, portal 10
00000100:00000001:2.0:1788575713.686637:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575713.686638:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:2.0:1788575713.686645:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.686652:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575713.686653:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575713.686656:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575713.686658:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393230208, offset 0
00000400:00000200:2.0:1788575713.686662:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575713.686666:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575713.686670:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575713.686673:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575713.687100:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575713.687105:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575713.687107:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575713.687111:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331054:331054:256:4294967295] 192.168.202.110@tcp LPNI seq info [331011:331011:8:4294967295]
00000400:00000200:2.0:1788575713.687115:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575713.687122:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575713.687127:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575713.687131:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575713.687136:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575713.687139:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575713.687144:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575713.687147:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575713.687150:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575713.687152:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575713.687155:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575713.687159:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575713.687163:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575713.687165:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575713.687167:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575713.687171:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.687179:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575713.687182:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.687188:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575713.687189:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575713.687193:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575713.687196:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575713.687199:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575713.687201:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307317287
00000100:00000001:2.0:1788575713.687204:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307318201
00000100:00000001:2.0:1788575713.687206:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307317287
00000100:00000001:2.0:1788575713.687208:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307318202
00000100:00000001:2.0:1788575713.687210:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575713.687212:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575713.687214:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575713.687217:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575713/real 1788575713]  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575713.687225:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575713.687226:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575713.687228:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575713.687230:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575713.687232:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575713.687234:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575713.687237:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575713.687245:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.687251:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575713.687252:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.687254:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575713.687255:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575713.687258:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.687264:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575713.687267:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.687273:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575713.687275:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.687276:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575713.687277:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.687279:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575713.687281:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575713.687283:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575713.687286:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575713.687288:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575713.687292:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575713.687299:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575713.687304:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575713.687307:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.687313:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575713.687314:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575713.687317:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575713.687318:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575713.687320:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575713.687323:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575713.687327:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575713.687329:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575713.687331:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575713.687332:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575713.687335:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575713.687337:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575713.687341:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575713.687344:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575713.687347:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575713.687350:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393230336, portal 4
00000100:00000001:2.0:1788575713.687352:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575713.687353:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788575713.687360:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.687365:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575713.687368:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575713.687370:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575713.687372:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393230336, offset 0
00000400:00000200:2.0:1788575713.687376:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575713.687380:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575713.687385:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575713.687388:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575713.687884:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575713.687890:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575713.687893:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575713.687896:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331055:331055:256:4294967295] 192.168.202.110@tcp LPNI seq info [331012:331012:8:4294967295]
00000400:00000200:2.0:1788575713.687901:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575713.687909:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575713.687915:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575713.687920:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575713.687927:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575713.687930:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575713.687934:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575713.687937:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575713.687941:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575713.687942:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575713.687946:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575713.687950:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575713.687955:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575713.687957:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575713.687960:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575713.687965:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.687974:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575713.687977:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.687982:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575713.687984:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575713.687988:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575713.687992:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575713.687995:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575713.687997:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307317434
00000100:00000001:2.0:1788575713.688000:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307318201
00000100:00000001:2.0:1788575713.688002:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307317434
00000100:00000001:2.0:1788575713.688005:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307318202
00000100:00000001:2.0:1788575713.688007:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575713.688009:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575713.688011:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575713.688014:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575713/real 1788575713]  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575713.688022:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575713.688023:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575713.688025:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575713.688027:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575713.688029:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575713.688031:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575713.688034:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575713.688042:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.688048:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575713.688049:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.688051:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575713.688053:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575713.688056:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.688062:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575713.688066:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.688072:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575713.688074:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.688076:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575713.688078:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.688080:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575713.688082:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575713.688085:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575713.688089:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575713.688091:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575713.688095:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575713.688101:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575713.688105:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.688109:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575713.688112:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393230080/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.688118:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575713.688130:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575713.688133:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575713.688136:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575713.688140:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575713.688141:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575713.688143:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575713.688147:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575713.688151:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575713.688152:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575713.688154:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788575713.688157:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575713.688159:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.688160:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575713.688164:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393230208/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.688169:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575713.688171:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575713.688172:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575713.688175:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575713.688176:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575713.688177:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575713.688179:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575713.688180:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575713.688182:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575713.688183:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575713.688184:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575713.688186:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575713.688187:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.688189:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575713.688192:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393230336/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575738 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575713.688198:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575713.688198:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575713.688200:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575713.688201:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575713.688202:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575713.688203:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575713.688205:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575713.688206:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575713.688208:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575713.688209:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575713.688210:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575713.688212:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575713.688213:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.688215:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575713.688226:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575713.688228:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575713.688230:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575713.688232:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575714.070741:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575714.070747:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575714.070750:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575714.070751:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575714.070753:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575714.070754:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575714.070818:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575714.070822:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575714.070825:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575714.197521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575714.197526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575714.197528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575714.197532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575714.197535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575714.197538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575714.197547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575714.197550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575714.197552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575714.645497:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307318201, at 4307318208
00000100:00000040:1.0:1788575714.645497:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307318202, at 4307318208
00000100:00000010:0.0:1788575714.645510:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00000100:00000010:1.0:1788575714.645516:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00010000:00000001:3.0:1788575715.093805:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575715.093811:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575715.093816:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575715.093819:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575715.093821:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575715.093825:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575715.093828:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575715.093830:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575715.093832:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575715.093834:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575715.093838:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575715.093839:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575715.093841:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575715.093842:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575715.093844:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575715.093845:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575715.093847:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575715.093848:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575715.093901:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575715.093905:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575715.093908:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575715.221545:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575715.221553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575715.221555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575715.221560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575715.221563:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575715.221567:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575715.221569:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575715.221572:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575715.221574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575716.117851:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575716.117856:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575716.117859:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575716.117860:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575716.117862:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575716.117863:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575716.117926:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575716.117929:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575716.117933:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575716.245540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575716.245546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575716.245548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575716.245551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575716.245554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575716.245557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575716.245561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575716.245563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575716.245570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575717.141702:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575717.141707:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575717.141711:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575717.141712:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575717.141714:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575717.141715:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575717.141765:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575717.141769:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575717.141772:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575717.269513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575717.269522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575717.269524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575717.269529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575717.269533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575717.269537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575717.269540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575717.269542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575717.269545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575718.165747:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575718.165755:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575718.165757:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575718.165758:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575718.165763:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575718.165764:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575718.165766:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575718.165768:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575718.165770:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575718.165771:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575718.165773:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575718.165774:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575718.165826:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575718.165830:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575718.165833:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575718.293499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575718.293507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575718.293509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575718.293522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575718.293526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575718.293528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575718.293530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575718.293532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575718.293538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575718.805556:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575718.805568:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575718.805570:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575718.805574:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575718.805578:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575718.805586:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12649
00000400:00000200:2.0:1788575718.805595:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575718.805601:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575718.805604:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575718.805608:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.805611:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575718.805613:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.805614:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575718.805617:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575718.805618:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575718.805621:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575718.805623:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575718.805624:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575718.805626:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575718.805628:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.805629:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575718.805631:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.805632:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575718.805634:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575718.805637:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575718.805638:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575718.805640:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575718.805642:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575718.805644:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575718.805646:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575718.805648:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575718.805657:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01180.
00000020:00000040:2.0:1788575718.805660:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575718.805664:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575718.805665:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575718.805668:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575718.805671:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575718.805674:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575718.805682:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000100:00000001:2.0:1788575718.805688:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575718.805697:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575718.805706:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575718.805710:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575718.805714:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b74282d7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575718.805732:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575718.805736:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575718.805739:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575718.805740:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575718.805742:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575718.805745:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575718.805747:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12649
00000400:00000200:2.0:1788575718.805752:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575718.805755:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575718.805757:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575718.805761:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.805763:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575718.805765:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.805767:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575718.805769:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575718.805771:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575718.805772:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575718.805773:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575718.805777:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575718.805777:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575718.805781:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788575718.805782:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575718.805786:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575718.805789:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575718.805792:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.805797:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575718.805802:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575718.805805:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575718.805806:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575718.805809:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575718.805809:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575718.805814:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575718.805817:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575718.805817:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575718.805820:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575718.805821:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575718.805825:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575718.805825:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575718.805830:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575718.805832:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393230464:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575718.805833:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575718.805839:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575718.805844:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575718.805846:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.805847:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:2.0:1788575718.805850:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03480.
02000000:00000001:3.0:1788575718.805853:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:2.0:1788575718.805855:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575718.805857:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575718.805860:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575718.805861:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1788575718.805863:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575718.805866:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575718.805868:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:3.0:1788575718.805869:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575718.805871:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575718.805876:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575718.805877:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11400.
00000100:00000001:2.0:1788575718.805883:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575718.805884:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575718.805891:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00100000:2.0:1788575718.805892:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575718.805900:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575718.805903:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575718.805907:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575718.805909:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393230464, portal 25
00000100:00000040:2.0:1788575718.805912:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9656041] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.805914:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575718.805916:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000001:2.0:1788575718.805926:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575718.805930:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575718.805932:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575718.805933:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575718.805936:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575718.805938:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575718.805943:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575718.805948:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12649
00000100:00000001:3.0:1788575718.805950:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575718.805955:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575718.805958:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575718.805962:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393230464, offset 0
00000400:00000200:3.0:1788575718.805969:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575718.805977:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575718.805982:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575718.805985:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575718.805988:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575718.805992:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575718.805994:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575718.805998:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331056:331056:256:4294967295] 192.168.202.110@tcp LPNI seq info [331013:331013:8:4294967295]
00000400:00000200:3.0:1788575718.806002:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575718.806009:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575718.806016:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1788575718.806018:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575718.806023:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1788575718.806029:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575718.806031:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.806034:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000010:3.0:1788575718.806036:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000100:00000040:2.0:1788575718.806037:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.806041:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575718.806045:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:3.0:1788575718.806046:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1788575718.806049:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:3.0:1788575718.806051:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000020:00000001:2.0:1788575718.806053:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.1:1788575718.806057:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1788575718.806058:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:3.1:1788575718.806060:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000020:00000001:2.0:1788575718.806063:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000100:3.0:1788575718.806066:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:2.0:1788575718.806070:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000010:3.0:1788575718.806070:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575718.806077:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1788575718.806078:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575718.806082:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1788575718.806083:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1788575718.806089:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575718.806089:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575718.806093:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:2.0:1788575718.806094:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575718.806099:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575718.806102:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575718.806104:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:2.0:1788575718.806105:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575718.806109:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1788575718.806111:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575718.806113:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575718.806118:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575718.806123:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575718.806125:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575718.806128:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575718.806130:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.806130:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575718.806136:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575743 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1788575718.806137:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000020:00000040:2.0:1788575718.806141:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575718.806145:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575718.806147:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575718.806151:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575718.806153:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1788575718.806154:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575718.806155:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.806158:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000010:2.0:1788575718.806160:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788575718.806164:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.806167:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575718.806168:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575718.806175:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575718.806180:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575718.806180:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307323321
00000100:00000001:2.0:1788575718.806182:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575718.806187:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1788575718.806188:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:2.0:1788575718.806195:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575718.806198:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575718.806198:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:2.0:1788575718.806203:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575718.806207:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12660)
00000100:00000001:3.0:1788575718.806207:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307323321
00000100:00000010:3.0:1788575718.806214:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575718.806218:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575718.806222:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575718.806226:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.806229:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575718.806234:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575718/real 1788575718]  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575718.806264:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575718.806265:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575718.806267:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575718.806270:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575718.806272:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575718.806274:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575718.806278:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575718.806285:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806291:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575718.806292:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806294:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575718.806296:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575718.806300:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806306:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575718.806310:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806316:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575718.806319:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806320:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575718.806322:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806324:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575718.806326:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575718.806329:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575718.806333:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575718.806335:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575718.806339:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575718.806348:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393230464:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575718.806352:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806354:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575718.806357:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393230464/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806363:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575718.806365:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575718.806368:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575718.806371:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575718.806372:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575718.806374:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575718.806376:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575718.806379:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575718.806382:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575718.806384:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575718.806385:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:3.0:1788575718.806388:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575718.806389:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806411:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806418:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575718.806419:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.806421:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575718.806423:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575718.806424:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575718.806436:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806443:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575718.806445:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575718.806447:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575718.806449:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575718.806451:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575718.806454:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230592:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575718.806457:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575718.806459:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575718.806461:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575718.806463:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575718.806466:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575718.806468:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575718.806472:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575718.806475:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575718.806479:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575718.806481:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393230592, portal 10
00000100:00000001:3.0:1788575718.806483:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575718.806485:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:3.0:1788575718.806502:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806509:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575718.806511:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575718.806514:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575718.806517:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393230592, offset 0
00000400:00000200:3.0:1788575718.806521:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575718.806525:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575718.806529:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575718.806532:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575718.806534:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575718.806537:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575718.806539:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575718.806542:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331057:331057:256:4294967295] 192.168.202.110@tcp LPNI seq info [331014:331014:8:4294967295]
00000400:00000200:3.0:1788575718.806546:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575718.806552:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575718.806556:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575718.806559:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575718.806562:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575718.806566:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1788575718.806570:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575718.806573:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:3.0:1788575718.806576:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575718.806578:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575718.806580:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575718.806583:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575718.806586:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575718.806587:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575718.806589:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575718.806591:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806596:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575718.806598:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575743 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806602:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.806603:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575718.806605:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575718.806608:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575718.806611:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575718.806613:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307322407
00000100:00000001:3.0:1788575718.806616:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307323321
00000100:00000001:3.0:1788575718.806618:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307322407
00000100:00000001:3.0:1788575718.806620:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307323321
00000100:00000001:3.0:1788575718.806622:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575718.806624:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.806626:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575718.806630:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575718/real 1788575718]  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575718.806637:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575718.806639:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575718.806641:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575718.806643:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575718.806645:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575718.806647:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575718.806650:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575718.806658:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806664:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575718.806665:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806667:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575718.806668:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575718.806671:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806677:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575718.806681:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806688:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575718.806690:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806692:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575718.806693:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806695:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575718.806697:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575718.806700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575718.806703:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575718.806705:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575718.806708:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575718.806713:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230592:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575718.806716:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575718.806718:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806722:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575718.806723:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575718.806724:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575718.806726:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575718.806727:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575718.806728:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230720:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575718.806730:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575718.806731:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575718.806732:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575718.806733:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575718.806735:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575718.806737:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575718.806739:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575718.806741:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575718.806743:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575718.806745:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393230720, portal 4
00000100:00000001:3.0:1788575718.806746:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575718.806747:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575718.806751:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806754:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575718.806756:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575718.806757:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575718.806758:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393230720, offset 0
00000400:00000200:3.0:1788575718.806760:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575718.806763:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575718.806765:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575718.806767:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575718.806770:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575718.806772:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575718.806775:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575718.806777:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331058:331058:256:4294967295] 192.168.202.110@tcp LPNI seq info [331015:331015:8:4294967295]
00000400:00000200:3.0:1788575718.806781:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575718.806787:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575718.806791:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575718.806795:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575718.806798:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575718.806801:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1788575718.806805:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575718.806807:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:3.0:1788575718.806811:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575718.806813:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575718.806815:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575718.806818:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575718.806821:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575718.806823:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575718.806825:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575718.806829:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575743 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806836:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575718.806838:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575743 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806844:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.806845:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575718.806848:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575718.806851:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575718.806854:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575718.806856:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307322553
00000100:00000001:3.0:1788575718.806858:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307323321
00000100:00000001:3.0:1788575718.806860:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307322553
00000100:00000001:3.0:1788575718.806863:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307323321
00000100:00000001:3.0:1788575718.806864:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575718.806867:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.806868:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575718.806872:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575718/real 1788575718]  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575718.806879:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575718.806880:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575718.806882:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575718.806883:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575718.806885:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575718.806887:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575718.806889:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575718.806896:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806902:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575718.806903:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806905:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575718.806906:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575718.806909:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806915:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575718.806918:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806924:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575718.806926:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806927:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575718.806928:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806930:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575718.806932:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575718.806934:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575718.806938:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575718.806940:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575718.806943:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575718.806951:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230720:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575718.806954:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806956:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575718.806959:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393230592/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806965:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575718.806966:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575718.806968:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575718.806970:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575718.806972:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575718.806973:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575718.806975:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575718.806976:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575718.806979:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575718.806980:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575718.806981:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:3.0:1788575718.806983:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575718.806984:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.806986:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575718.806989:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393230720/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575743 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575718.806995:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575718.806996:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575718.806998:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575718.807000:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575718.807001:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575718.807002:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575718.807004:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575718.807005:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575718.807007:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575718.807008:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575718.807009:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575718.807011:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575718.807012:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.807014:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575718.807017:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575718.807019:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575718.807020:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575718.807022:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575719.189521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575719.189536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575719.189542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575719.189543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575719.189546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575719.189548:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575719.189611:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575719.189615:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575719.189618:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575719.317554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575719.317562:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575719.317564:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575719.317570:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575719.317573:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575719.317577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575719.317579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575719.317582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575719.317585:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575719.765540:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307323321, at 4307323328
00000100:00000010:0.0:1788575719.765555:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000040:1.0:1788575719.767065:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307323321, at 4307323329
00000100:00000010:1.0:1788575719.767073:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575720.213543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575720.213549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575720.213555:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575720.213556:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575720.213560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575720.213562:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575720.213659:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575720.213662:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575720.213665:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575720.341555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575720.341562:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575720.341564:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575720.341569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575720.341572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575720.341580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575720.341582:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575720.341584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575720.341588:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575721.237513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575721.237565:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575721.237570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575721.237571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575721.237573:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575721.237574:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575721.237632:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575721.237635:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575721.237638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575721.365509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575721.365527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575721.365529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575721.365535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575721.365539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575721.365542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575721.365545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575721.365548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575721.365551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575722.261521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575722.261529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575722.261536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575722.261538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575722.261543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575722.261545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575722.261583:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575722.261588:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575722.261592:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575722.389548:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575722.389559:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575722.389562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575722.389568:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575722.389573:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575722.389577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575722.389579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575722.389581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575722.389583:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575723.285501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575723.285508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575723.285513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575723.285514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575723.285517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575723.285518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575723.285587:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575723.285591:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575723.285595:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575723.413502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575723.413511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575723.413514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575723.413520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575723.413523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575723.413527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575723.413529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575723.413531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575723.413534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575723.925499:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575723.925509:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575723.925512:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575723.925516:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575723.925521:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575723.925528:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12655
00000400:00000200:2.0:1788575723.925537:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575723.925542:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575723.925545:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575723.925547:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925550:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575723.925551:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925554:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575723.925557:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575723.925558:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575723.925561:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575723.925563:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575723.925565:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575723.925568:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575723.925570:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925571:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575723.925572:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925573:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575723.925575:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575723.925578:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925579:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575723.925582:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575723.925585:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575723.925587:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575723.925588:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575723.925590:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575723.925600:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00700.
00000020:00000040:2.0:1788575723.925603:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575723.925606:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575723.925607:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575723.925609:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575723.925612:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575723.925616:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575723.925624:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000100:00000001:2.0:1788575723.925630:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575723.925638:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575723.925647:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575723.925651:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575723.925654:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e69941e] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575723.925675:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575723.925677:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575723.925680:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575723.925682:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575723.925683:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575723.925685:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575723.925687:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12655
00000400:00000200:2.0:1788575723.925695:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575723.925702:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575723.925704:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575723.925708:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925711:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575723.925713:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925715:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575723.925718:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1788575723.925718:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1788575723.925721:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575723.925724:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575723.925725:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1788575723.925728:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575723.925730:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788575723.925732:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575723.925736:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575723.925740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575723.925743:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925748:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575723.925752:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925756:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575723.925759:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1788575723.925761:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575723.925763:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575723.925766:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575723.925769:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1788575723.925770:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575723.925772:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575723.925774:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575723.925775:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575723.925780:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393230848:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575723.925783:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575723.925784:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575723.925787:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575723.925788:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575723.925790:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575723.925792:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575723.925793:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575723.925799:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
00000008:00100000:2.0:1788575723.925800:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575723.925801:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788575723.925807:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575723.925811:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575723.925813:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575723.925816:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000010:2.0:1788575723.925819:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02680.
00000020:00000040:2.0:1788575723.925822:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:3.0:1788575723.925823:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000001:2.0:1788575723.925825:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575723.925828:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575723.925830:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:3.0:1788575723.925831:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393230848, portal 25
02000000:00000001:2.0:1788575723.925832:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575723.925836:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.925836:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575723.925839:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
02000000:00000010:2.0:1788575723.925843:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000040:3.0:1788575723.925845:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575723.925850:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.925852:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575723.925856:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00100000:2.0:1788575723.925861:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575723.925861:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575723.925867:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393230848, offset 0
00000100:00000001:2.0:1788575723.925874:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575723.925875:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925880:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575723.925885:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000450d1599] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575723.925887:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575723.925896:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1788575723.925901:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575723.925903:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1788575723.925905:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575723.925909:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1788575723.925910:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575723.925913:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575723.925914:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575723.925916:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1788575723.925918:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:3.0:1788575723.925919:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:2.0:1788575723.925926:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12655
00000400:00000200:3.0:1788575723.925926:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331059:331059:256:4294967295] 192.168.202.110@tcp LPNI seq info [331016:331016:8:4294967295]
00000400:00000200:3.0:1788575723.925933:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575723.925938:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575723.925944:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1788575723.925947:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575723.925952:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575723.925957:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925959:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575723.925963:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575723.925963:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925969:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575723.925971:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000010:3.0:1788575723.925972:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000020:00000040:2.0:1788575723.925974:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575723.925977:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575723.925980:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575723.925984:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.0:1788575723.925984:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1788575723.925990:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.0:1788575723.925992:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000040:2.0:1788575723.925995:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575723.925999:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1788575723.926003:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575723.926005:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575723.926007:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:2.0:1788575723.926010:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000100:3.0:1788575723.926014:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:2.0:1788575723.926016:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000010:3.0:1788575723.926017:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000100:00080000:2.0:1788575723.926022:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575723.926025:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575723.926026:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1788575723.926027:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575723.926032:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1788575723.926033:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575723.926037:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575723.926038:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000100:00000001:2.0:1788575723.926039:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1788575723.926042:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000008:00100000:2.0:1788575723.926044:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575723.926048:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926049:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:2.0:1788575723.926050:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575723.926055:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1788575723.926057:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01f80.
00000020:00000040:2.0:1788575723.926062:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575723.926067:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575723.926070:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575723.926074:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575723.926076:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1788575723.926076:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1788575723.926081:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575723.926083:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788575723.926088:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788575723.926094:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926098:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926101:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00100000:2.0:1788575723.926103:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575723.926109:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575723.926114:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575723.926117:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575723.926118:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926121:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307328441
00000100:00000040:2.0:1788575723.926125:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8c8ec7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1788575723.926129:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:2.0:1788575723.926135:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575723.926137:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575723.926138:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:2.0:1788575723.926143:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575723.926147:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307328441
00000100:00000040:2.0:1788575723.926149:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12665)
00000100:00000010:3.0:1788575723.926152:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575723.926156:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575723.926159:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575723.926161:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926163:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575723.926168:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575723/real 1788575723]  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575723.926176:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575723.926177:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575723.926179:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575723.926180:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575723.926182:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575723.926184:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575723.926186:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575723.926192:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926196:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575723.926197:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926198:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575723.926200:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575723.926202:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926206:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575723.926209:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926214:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575723.926216:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926217:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575723.926218:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926220:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575723.926221:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575723.926223:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575723.926225:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575723.926227:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575723.926229:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575723.926234:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393230848:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575723.926237:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926260:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575723.926263:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393230848/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926267:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575723.926270:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575723.926272:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575723.926274:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575723.926276:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575723.926277:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575723.926279:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575723.926281:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575723.926284:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575723.926286:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575723.926287:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575723.926289:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575723.926290:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926292:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926300:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575723.926301:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926302:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575723.926303:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575723.926304:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575723.926307:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926311:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575723.926312:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575723.926314:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575723.926315:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575723.926316:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575723.926318:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230976:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575723.926320:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575723.926321:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575723.926322:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575723.926323:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575723.926325:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575723.926327:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575723.926329:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575723.926331:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575723.926334:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575723.926336:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393230976, portal 10
00000100:00000001:3.0:1788575723.926337:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575723.926338:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:3.0:1788575723.926343:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926347:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575723.926348:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575723.926350:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575723.926351:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393230976, offset 0
00000400:00000200:3.0:1788575723.926353:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575723.926356:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575723.926360:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575723.926363:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575723.926365:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575723.926368:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575723.926370:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575723.926373:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331060:331060:256:4294967295] 192.168.202.110@tcp LPNI seq info [331017:331017:8:4294967295]
00000400:00000200:3.0:1788575723.926377:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575723.926384:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575723.926389:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575723.926414:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575723.926418:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575723.926419:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1788575723.926422:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575723.926424:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1788575723.926426:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575723.926428:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575723.926430:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575723.926432:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575723.926434:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575723.926436:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575723.926437:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575723.926440:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926445:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575723.926446:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926451:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926452:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575723.926454:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575723.926457:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575723.926459:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575723.926460:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307327527
00000100:00000001:3.0:1788575723.926463:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307328441
00000100:00000001:3.0:1788575723.926464:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307327527
00000100:00000001:3.0:1788575723.926466:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307328441
00000100:00000001:3.0:1788575723.926468:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575723.926470:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926471:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575723.926475:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575723/real 1788575723]  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575723.926483:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575723.926484:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575723.926486:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575723.926488:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575723.926490:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575723.926492:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575723.926495:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575723.926504:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926509:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575723.926510:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926512:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575723.926514:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575723.926517:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926523:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575723.926526:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926532:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575723.926534:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926536:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575723.926538:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926539:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575723.926542:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575723.926544:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575723.926547:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575723.926549:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575723.926552:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575723.926559:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393230976:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575723.926563:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575723.926566:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926571:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575723.926573:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575723.926574:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575723.926576:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575723.926578:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575723.926580:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575723.926583:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575723.926585:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575723.926586:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575723.926588:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575723.926591:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575723.926593:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575723.926596:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575723.926599:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575723.926602:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575723.926604:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393231104, portal 4
00000100:00000001:3.0:1788575723.926606:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575723.926608:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:3.0:1788575723.926613:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926627:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575723.926630:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575723.926632:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575723.926634:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393231104, offset 0
00000400:00000200:3.0:1788575723.926638:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575723.926642:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575723.926645:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575723.926648:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575723.926650:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575723.926652:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575723.926654:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575723.926657:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331061:331061:256:4294967295] 192.168.202.110@tcp LPNI seq info [331018:331018:8:4294967295]
00000400:00000200:3.0:1788575723.926661:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575723.926667:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575723.926671:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575723.926673:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575723.926677:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575723.926680:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1788575723.926683:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575723.926685:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1788575723.926689:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575723.926691:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575723.926693:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575723.926696:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575723.926699:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575723.926700:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575723.926703:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575723.926706:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926713:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575723.926716:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926722:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926724:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575723.926726:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575723.926729:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575723.926732:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575723.926734:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307327673
00000100:00000001:3.0:1788575723.926736:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307328441
00000100:00000001:3.0:1788575723.926738:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307327673
00000100:00000001:3.0:1788575723.926739:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307328441
00000100:00000001:3.0:1788575723.926741:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575723.926743:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926745:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575723.926748:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575723/real 1788575723]  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575723.926755:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575723.926756:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575723.926757:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575723.926760:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575723.926761:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575723.926763:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575723.926765:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575723.926772:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926778:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575723.926779:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926781:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575723.926782:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575723.926786:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926792:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575723.926795:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926801:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575723.926803:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926804:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575723.926805:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926807:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575723.926809:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575723.926811:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575723.926814:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575723.926816:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575723.926819:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575723.926826:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575723.926829:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926831:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575723.926834:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393230976/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926839:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575723.926841:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575723.926842:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575723.926844:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575723.926846:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575723.926847:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575723.926849:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575723.926851:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575723.926853:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575723.926855:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575723.926856:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575723.926858:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575723.926867:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926869:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575723.926874:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393231104/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575723.926880:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575723.926881:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575723.926883:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575723.926885:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575723.926886:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575723.926887:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575723.926888:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575723.926890:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575723.926892:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575723.926893:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575723.926894:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575723.926896:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575723.926897:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926899:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575723.926902:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575723.926903:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575723.926905:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575723.926907:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575724.309529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575724.309539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575724.309545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575724.309546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575724.309548:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575724.309549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575724.309603:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575724.309607:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575724.309610:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575724.437555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575724.437563:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575724.437565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575724.437571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575724.437574:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575724.437579:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575724.437581:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575724.437583:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575724.437586:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575724.885518:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307328441, at 4307328448
00000100:00000040:1.0:1788575724.885518:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307328441, at 4307328448
00000100:00000010:0.0:1788575724.885539:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:1.0:1788575724.885545:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575725.333501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575725.333506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575725.333512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575725.333515:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575725.333518:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575725.333522:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575725.333525:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575725.333527:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575725.333530:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575725.333532:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575725.333535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575725.333537:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575725.333538:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575725.333539:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575725.333540:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575725.333542:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575725.333543:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575725.333545:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575725.333633:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575725.333639:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575725.333643:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575725.461533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575725.461542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575725.461545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575725.461552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575725.461557:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575725.461561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575725.461564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575725.461567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575725.461571:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575726.357489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575726.357495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575726.357500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575726.357501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575726.357505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575726.357506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575726.357569:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575726.357574:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575726.357577:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575726.485519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575726.485525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575726.485527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575726.485532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575726.485536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575726.485540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575726.485543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575726.485544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575726.485546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575727.381509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575727.381516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575727.381522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575727.381523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575727.381527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575727.381528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575727.381581:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575727.381584:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575727.381588:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575727.509467:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575727.509472:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575727.509473:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575727.509477:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575727.509479:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575727.509481:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575727.509483:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575727.509485:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575727.509487:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575728.405495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575728.405502:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575728.405505:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575728.405507:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575728.405510:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575728.405512:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575728.405513:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575728.405515:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575728.405519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575728.405520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575728.405524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575728.405525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575728.405612:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575728.405616:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575728.405619:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575728.533498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575728.533504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575728.533506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575728.533511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575728.533515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575728.533518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575728.533521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575728.533523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575728.533526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575729.045487:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575729.045498:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575729.045501:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575729.045505:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575729.045509:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575729.045518:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12660
00000400:00000200:2.0:1788575729.045527:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575729.045533:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575729.045536:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575729.045538:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045541:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575729.045542:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045544:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575729.045547:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575729.045548:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575729.045550:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575729.045552:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575729.045554:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575729.045556:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575729.045558:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045559:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575729.045560:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045561:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575729.045563:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575729.045565:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045567:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575729.045569:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575729.045571:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575729.045573:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.045574:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575729.045576:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575729.045583:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02a00.
00000020:00000040:2.0:1788575729.045585:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575729.045588:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575729.045589:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.045591:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575729.045592:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575729.045596:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575729.045604:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788575729.045611:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575729.045620:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575729.045629:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575729.045634:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575729.045638:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cce910e4] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575729.045662:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575729.045665:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575729.045668:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575729.045670:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575729.045672:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575729.045675:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575729.045677:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12660
00000400:00000200:2.0:1788575729.045682:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575729.045686:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575729.045689:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575729.045691:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1788575729.045692:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575729.045695:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575729.045696:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575729.045698:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575729.045701:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045705:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:3.0:1788575729.045708:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575729.045709:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575729.045712:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575729.045715:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575729.045719:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575729.045724:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575729.045726:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575729.045726:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575729.045730:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575729.045734:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575729.045737:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575729.045741:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575729.045741:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.045747:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1788575729.045750:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393231232:192.168.202.110@tcp:250:kworker.0
00000100:00000040:2.0:1788575729.045753:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045759:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575729.045759:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1788575729.045764:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575729.045765:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:2.0:1788575729.045768:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575729.045770:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.045773:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575729.045775:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1788575729.045777:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:3.0:1788575729.045781:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575729.045783:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575729.045786:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575729.045786:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.045791:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575729.045795:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1788575729.045797:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:2.0:1788575729.045798:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.045801:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575729.045804:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000010:2.0:1788575729.045805:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03800.
00000020:00000040:2.0:1788575729.045808:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575729.045810:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575729.045811:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.045812:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1788575729.045813:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000001:2.0:1788575729.045814:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575729.045816:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575729.045821:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000200:3.0:1788575729.045823:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393231232, portal 25
00000100:00000001:2.0:1788575729.045824:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.045828:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575729.045831:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00100000:2.0:1788575729.045833:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1788575729.045840:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575729.045845:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575729.045849:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.045855:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575729.045859:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014d98894] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575729.045862:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575729.045867:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575729.045872:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393231232, offset 0
00000100:00000001:2.0:1788575729.045874:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575729.045878:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575729.045880:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575729.045883:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575729.045884:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575729.045884:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045889:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575729.045892:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12660
00000400:00000200:3.0:1788575729.045895:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575729.045903:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575729.045908:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575729.045918:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575729.045922:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575729.045925:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575729.045928:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331062:331062:256:4294967295] 192.168.202.110@tcp LPNI seq info [331019:331019:8:4294967295]
00000400:00000200:3.0:1788575729.045933:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575729.045938:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575729.045942:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575729.045946:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575729.045950:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575729.045952:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1788575729.045953:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575729.045957:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045960:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575729.045962:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575729.045964:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575729.045965:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575729.045966:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575729.045967:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575729.045968:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575729.045972:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045974:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575729.045974:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575729.045978:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575729.045980:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000010:3.0:1788575729.045981:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000020:00000040:2.0:1788575729.045982:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575729.045986:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575729.046003:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575729.046004:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1788575729.046006:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.0:1788575729.046010:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
02000000:00000001:2.0:1788575729.046011:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575729.046016:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.046021:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.1:1788575729.046021:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000008:00100000:2.0:1788575729.046025:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:3.1:1788575729.046026:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000008:00000001:2.0:1788575729.046029:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.046033:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1788575729.046035:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000010:2.0:1788575729.046039:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00a80.
00000800:00000010:3.0:1788575729.046039:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000020:00000040:2.0:1788575729.046044:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575729.046046:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1788575729.046047:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575729.046049:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575729.046051:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575729.046054:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575729.046054:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1788575729.046056:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575729.046060:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
02000000:00000010:2.0:1788575729.046061:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000400:00000010:3.0:1788575729.046064:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:2.0:1788575729.046065:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046069:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00100000:2.0:1788575729.046072:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575729.046076:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575729.046088:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575729.046091:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046093:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575729.046096:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000389a8694] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575729.046098:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575754 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575729.046118:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046120:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575729.046121:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575729.046124:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:2.0:1788575729.046126:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575729.046131:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12670)
00000100:00000001:3.0:1788575729.046135:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575729.046141:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575729.046143:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307333561
00000100:00000010:3.0:1788575729.046149:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788575729.046156:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575729.046161:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307333561
00000100:00000010:3.0:1788575729.046165:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575729.046168:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575729.046171:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575729.046174:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046176:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575729.046181:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575729/real 1788575729]  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575729.046191:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575729.046192:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575729.046195:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575729.046197:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575729.046199:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575729.046201:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575729.046205:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575729.046214:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046221:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575729.046223:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046225:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575729.046227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575729.046232:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046238:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575729.046242:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046249:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575729.046252:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046253:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575729.046254:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046256:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575729.046257:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575729.046268:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575729.046272:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575729.046275:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575729.046279:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575729.046287:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393231232:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575729.046291:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046293:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575729.046296:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393231232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046303:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575729.046305:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575729.046307:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575729.046310:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575729.046311:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575729.046313:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575729.046315:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575729.046317:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575729.046327:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575729.046330:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575729.046335:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:3.0:1788575729.046338:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575729.046340:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046342:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046348:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575729.046350:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046352:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575729.046353:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575729.046355:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575729.046359:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046371:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575729.046372:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575729.046374:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575729.046376:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575729.046378:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575729.046381:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575729.046385:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575729.046387:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575729.046388:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575729.046413:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575729.046417:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575729.046419:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575729.046423:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575729.046426:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575729.046429:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575729.046432:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393231360, portal 10
00000100:00000001:3.0:1788575729.046434:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575729.046436:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:3.0:1788575729.046442:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046449:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575729.046451:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575729.046454:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575729.046456:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393231360, offset 0
00000400:00000200:3.0:1788575729.046460:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575729.046464:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575729.046468:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575729.046471:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575729.046473:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575729.046475:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575729.046478:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575729.046481:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331063:331063:256:4294967295] 192.168.202.110@tcp LPNI seq info [331020:331020:8:4294967295]
00000400:00000200:3.0:1788575729.046485:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575729.046491:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575729.046495:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575729.046499:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575729.046502:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575729.046505:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788575729.046509:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575729.046512:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1788575729.046516:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575729.046517:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575729.046520:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575729.046524:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575729.046527:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575729.046529:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575729.046531:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575729.046534:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046542:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575729.046545:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575754 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046552:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046553:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575729.046556:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575729.046559:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575729.046562:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575729.046564:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307332647
00000100:00000001:3.0:1788575729.046567:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307333561
00000100:00000001:3.0:1788575729.046569:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307332647
00000100:00000001:3.0:1788575729.046571:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307333561
00000100:00000001:3.0:1788575729.046573:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575729.046575:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046577:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575729.046580:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575729/real 1788575729]  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575729.046587:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575729.046588:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575729.046590:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575729.046593:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575729.046594:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575729.046596:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575729.046599:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575729.046616:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046622:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575729.046624:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046626:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575729.046627:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575729.046631:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046637:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575729.046640:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046646:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575729.046648:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046650:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575729.046651:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046653:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575729.046655:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575729.046658:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575729.046661:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575729.046663:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575729.046667:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575729.046676:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575729.046679:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575729.046683:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046690:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575729.046691:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575729.046693:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575729.046695:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575729.046697:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575729.046699:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231488:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575729.046702:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575729.046704:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575729.046706:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575729.046707:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575729.046710:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575729.046712:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575729.046716:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575729.046718:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575729.046721:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575729.046724:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393231488, portal 4
00000100:00000001:3.0:1788575729.046726:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575729.046727:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:3.0:1788575729.046732:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046738:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575729.046741:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575729.046743:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575729.046745:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393231488, offset 0
00000400:00000200:3.0:1788575729.046749:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575729.046752:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575729.046756:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575729.046758:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575729.046761:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575729.046763:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575729.046765:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575729.046769:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331064:331064:256:4294967295] 192.168.202.110@tcp LPNI seq info [331021:331021:8:4294967295]
00000400:00000200:3.0:1788575729.046772:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575729.046779:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575729.046783:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575729.046785:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575729.046787:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575729.046789:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1788575729.046791:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575729.046793:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1788575729.046795:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575729.046796:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575729.046798:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575729.046800:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575729.046802:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575729.046803:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575729.046805:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575729.046807:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046812:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575729.046814:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575754 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046818:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046819:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575729.046821:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575729.046823:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575729.046825:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575729.046826:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307332793
00000100:00000001:3.0:1788575729.046828:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307333561
00000100:00000001:3.0:1788575729.046829:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307332793
00000100:00000001:3.0:1788575729.046830:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307333561
00000100:00000001:3.0:1788575729.046832:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575729.046833:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046834:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575729.046836:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575729/real 1788575729]  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575729.046841:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575729.046841:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575729.046842:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575729.046844:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575729.046845:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575729.046846:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575729.046847:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575729.046852:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046855:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575729.046856:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046857:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575729.046859:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575729.046862:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046867:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575729.046871:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046877:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575729.046879:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046881:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575729.046882:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046884:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575729.046885:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575729.046888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575729.046891:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575729.046893:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575729.046896:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575729.046904:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231488:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575729.046907:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046909:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575729.046912:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393231360/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046919:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575729.046920:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575729.046922:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575729.046925:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575729.046926:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575729.046927:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575729.046929:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575729.046931:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575729.046933:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575729.046935:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575729.046936:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:3.0:1788575729.046939:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575729.046940:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046942:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575729.046945:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393231488/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575754 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575729.046951:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575729.046952:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575729.046954:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575729.046955:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575729.046957:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575729.046958:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575729.046959:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575729.046960:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575729.046961:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575729.046962:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575729.046963:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575729.046964:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575729.046965:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046966:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575729.046969:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575729.046969:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575729.046971:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575729.046971:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575729.429500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575729.429507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575729.429512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575729.429514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575729.429518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575729.429519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575729.430034:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575729.430037:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575729.430040:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575729.557474:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575729.557482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575729.557484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575729.557488:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575729.557492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575729.557495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575729.557498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575729.557500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575729.557502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575730.005583:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307333561, at 4307333568
00000100:00000010:0.0:1788575730.005599:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00000100:00000040:1.0:1788575730.008424:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307333561, at 4307333571
00000100:00000010:1.0:1788575730.008435:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575730.453500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575730.453513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575730.453518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575730.453519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575730.453522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575730.453524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575730.453628:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575730.453632:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575730.453635:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575730.581508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575730.581516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575730.581518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575730.581524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575730.581528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575730.581531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575730.581534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575730.581537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575730.581540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575731.477510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575731.477517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575731.477524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575731.477525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575731.477529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575731.477531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575731.477623:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575731.477627:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575731.477630:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575731.605494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575731.605502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575731.605504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575731.605510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575731.605513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575731.605516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575731.605519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575731.605522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575731.605525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575732.501525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575732.501532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575732.501537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575732.501538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575732.501542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575732.501543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575732.501610:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575732.501614:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575732.501617:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575732.629512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575732.629521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575732.629523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575732.629529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575732.629533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575732.629537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575732.629539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575732.629542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575732.629545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575733.525662:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575733.525668:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575733.525850:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575733.525851:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575733.525855:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575733.525857:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575733.525957:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575733.525960:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575733.525963:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575733.653619:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575733.653625:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575733.653626:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575733.653630:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575733.653632:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575733.653634:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575733.653636:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575733.653638:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575733.653639:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575734.165774:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575734.165785:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575734.165788:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575734.165792:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575734.165797:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575734.165804:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12665
00000400:00000200:0.0:1788575734.165812:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575734.165819:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575734.165823:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575734.165826:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.165829:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575734.165831:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.165834:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575734.165838:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575734.165840:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575734.165842:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575734.165845:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575734.165847:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575734.165849:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575734.165852:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.165854:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575734.165857:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.165858:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575734.165861:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575734.165865:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575734.165867:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575734.165870:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575734.165872:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575734.165875:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575734.165877:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575734.165879:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575734.165888:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788575734.165891:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575734.165895:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575734.165897:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575734.165900:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575734.165902:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575734.165905:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575734.165912:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788575734.165930:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575734.165940:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575734.165950:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575734.165954:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575734.165958:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575734.165977:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575734.165980:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575734.165983:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575734.165984:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575734.165986:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575734.165989:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575734.165992:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12665
00000400:00000200:0.0:1788575734.165997:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575734.166001:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575734.166004:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575734.166007:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.166010:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575734.166010:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1788575734.166014:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575734.166016:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575734.166017:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575734.166020:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1788575734.166021:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575734.166024:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575734.166026:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575734.166028:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000040:3.0:1788575734.166031:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1788575734.166032:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575734.166035:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575734.166042:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.166046:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575734.166050:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575734.166052:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575734.166056:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575734.166057:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575734.166061:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575734.166062:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1788575734.166066:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575734.166067:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575734.166071:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575734.166076:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575734.166076:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393231616:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788575734.166080:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575734.166084:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575734.166087:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575734.166090:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575734.166090:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575734.166094:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575734.166095:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575734.166099:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1788575734.166100:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575734.166104:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575734.166106:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575734.166106:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575734.166111:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575734.166114:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575734.166118:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:3.0:1788575734.166120:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575734.166122:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575734.166125:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575734.166127:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:0.0:1788575734.166128:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575734.166130:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575734.166133:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575734.166136:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000010:0.0:1788575734.166138:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000200:3.0:1788575734.166144:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393231616, portal 25
00000100:00000001:3.0:1788575734.166148:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575734.166151:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.166152:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00100000:0.0:1788575734.166161:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1788575734.166163:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575734.166181:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575734.166184:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788575734.166185:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575734.166190:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788575734.166193:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788575734.166196:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575734.166202:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393231616, offset 0
00000100:00000001:0.0:1788575734.166209:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575734.166214:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575734.166222:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575734.166228:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575734.166233:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575734.166237:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575734.166241:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575734.166243:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575734.166247:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331065:331065:256:4294967295] 192.168.202.110@tcp LPNI seq info [331022:331022:8:4294967295]
00000400:00000200:3.0:1788575734.166251:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575734.166261:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575734.166266:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575734.166271:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575734.166278:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575734.166282:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788575734.166287:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575734.166290:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1788575734.166294:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575734.166296:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575734.166302:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575734.166312:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575734.166318:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575734.166320:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575734.166324:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575734.166327:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166336:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575734.166340:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575759 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166346:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.166347:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575734.166353:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575734.166357:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575734.166360:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307338681
00000100:00000010:3.0:1788575734.166365:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575734.166372:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575734.166377:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307338681
00000100:00000010:3.0:1788575734.166380:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575734.166383:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575734.166386:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575734.166389:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.166411:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575734.166416:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575734/real 1788575734]  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575734.166425:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575734.166426:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575734.166429:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575734.166431:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575734.166433:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575734.166435:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575734.166439:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575734.166447:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166453:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575734.166455:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166457:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575734.166459:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575734.166463:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166471:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575734.166475:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166481:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575734.166484:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166485:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575734.166487:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166489:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575734.166490:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575734.166493:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575734.166511:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575734.166513:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575734.166518:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575734.166525:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393231616:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575734.166529:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166532:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575734.166535:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393231616/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166541:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575734.166543:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575734.166555:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575734.166558:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575734.166560:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575734.166561:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575734.166563:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575734.166566:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5c00.
02000000:00000001:3.0:1788575734.166569:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575734.166571:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575734.166573:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:3.0:1788575734.166576:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575734.166577:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166579:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166586:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575734.166588:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.166589:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575734.166591:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575734.166592:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575734.166596:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166602:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575734.166604:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575734.166606:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575734.166607:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575734.166609:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575734.166611:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231744:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575734.166614:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575734.166616:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575734.166618:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575734.166619:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575734.166622:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575734.166623:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575734.166627:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575734.166630:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575734.166633:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575734.166636:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393231744, portal 10
00000100:00000001:3.0:1788575734.166637:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575734.166639:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00000040:3.0:1788575734.166645:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166651:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575734.166654:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575734.166657:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575734.166659:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393231744, offset 0
00000400:00000200:3.0:1788575734.166663:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575734.166666:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575734.166670:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575734.166673:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575734.166675:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575734.166677:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575734.166679:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575734.166682:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331066:331066:256:4294967295] 192.168.202.110@tcp LPNI seq info [331023:331023:8:4294967295]
00000400:00000200:3.0:1788575734.166686:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575734.166692:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575734.166696:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575734.166699:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575734.166703:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575734.166705:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788575734.166709:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575734.166711:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1788575734.166715:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575734.166717:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575734.166719:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575734.166723:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575734.166726:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575734.166727:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575734.166730:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575734.166733:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166752:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575734.166756:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575759 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166763:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.166764:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575734.166767:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575734.166770:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575734.166774:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575734.166776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307337767
00000100:00000001:3.0:1788575734.166778:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307338681
00000100:00000001:3.0:1788575734.166780:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307337767
00000100:00000001:3.0:1788575734.166783:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307338681
00000100:00000001:3.0:1788575734.166785:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575734.166787:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.166789:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575734.166792:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575734/real 1788575734]  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575734.166799:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575734.166801:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575734.166802:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575734.166805:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575734.166806:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575734.166808:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575734.166811:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575734.166819:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166825:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575734.166826:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166827:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575734.166829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575734.166832:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166838:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575734.166841:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166848:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575734.166851:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166852:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575734.166853:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166855:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575734.166857:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575734.166859:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575734.166862:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575734.166865:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575734.166868:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575734.166876:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231744:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575734.166879:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166881:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575734.166883:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393231744/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.166889:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575734.166890:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575734.166892:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575734.166895:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575734.166896:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575734.166897:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575734.166899:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575734.166901:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:3.0:1788575734.166903:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575734.166904:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575734.166905:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:3.0:1788575734.166908:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575734.166909:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166911:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.166914:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575734.166915:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.166917:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575734.166918:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575734.167069:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575734.167074:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575734.167076:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575734.167078:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575734.167081:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575734.167084:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12665
00000400:00000200:0.0:1788575734.167088:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575734.167091:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575734.167093:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575734.167096:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.167098:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575734.167100:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.167102:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575734.167104:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575734.167105:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575734.167107:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575734.167109:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575734.167111:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575734.167114:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575734.167116:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.167118:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575734.167120:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575734.167122:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575734.167124:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575734.167126:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575734.167128:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575734.167130:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575734.167133:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575734.167135:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575734.167137:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575734.167139:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575734.167142:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575734.167144:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575734.167147:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788575734.167149:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575734.167152:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575734.167153:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575734.167155:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575734.167156:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575734.167158:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575734.167162:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788575734.167165:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575734.167171:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575734.167178:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575734.167181:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575734.167184:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575734.167203:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575734.167207:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575734.167210:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575734.167215:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12675)
00000100:00000001:3.0:1788575734.167682:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575734.167684:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575734.167685:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575734.167690:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.167698:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575734.167699:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575734.167701:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575734.167703:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575734.167705:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575734.167708:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231872:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575734.167711:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575734.167713:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575734.167715:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575734.167716:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575734.167720:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575734.167721:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575734.167726:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575734.167728:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575734.167732:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575734.167735:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393231872, portal 4
00000100:00000001:3.0:1788575734.167736:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575734.167738:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:3.0:1788575734.167758:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.167764:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575734.167766:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575734.167768:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575734.167771:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393231872, offset 0
00000400:00000200:3.0:1788575734.167774:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575734.167778:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575734.167782:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575734.167785:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575734.167787:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575734.167789:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575734.167791:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575734.167794:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331067:331067:256:4294967295] 192.168.202.110@tcp LPNI seq info [331024:331024:8:4294967295]
00000400:00000200:3.0:1788575734.167798:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575734.167805:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575734.167809:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575734.167813:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575734.167816:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575734.167819:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1788575734.167823:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575734.167826:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1788575734.167830:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575734.167831:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575734.167834:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575734.167838:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575734.167841:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575734.167842:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575734.167844:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575734.167848:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575759 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.167855:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575734.167857:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575759 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.167864:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.167865:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575734.167868:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575734.167871:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575734.167874:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575734.167876:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307337914
00000100:00000001:3.0:1788575734.167878:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307338681
00000100:00000001:3.0:1788575734.167881:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307337914
00000100:00000001:3.0:1788575734.167883:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307338681
00000100:00000001:3.0:1788575734.167885:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575734.167886:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.167888:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575734.167891:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575734/real 1788575734]  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575734.167898:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575734.167900:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575734.167901:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575734.167903:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575734.167905:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575734.167906:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575734.167909:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575734.167917:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.167922:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575734.167924:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.167925:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575734.167927:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575734.167930:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.167937:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575734.167940:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.167947:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575734.167950:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.167951:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575734.167952:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.167954:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575734.167956:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575734.167958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575734.167961:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575734.167963:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575734.167967:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575734.167974:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393231872:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575734.167978:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.167979:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575734.167982:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393231872/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575759 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575734.167988:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575734.167990:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575734.167992:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575734.167994:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575734.167995:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575734.167997:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575734.167998:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575734.168000:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:3.0:1788575734.168003:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575734.168004:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575734.168006:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:3.0:1788575734.168008:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575734.168009:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.168011:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575734.168014:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575734.168015:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575734.168017:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575734.168019:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575734.549512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575734.549519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575734.549524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575734.549525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575734.549528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575734.549528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575734.549586:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575734.549591:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575734.549594:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575734.677502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575734.677508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575734.677510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575734.677514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575734.677517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575734.677520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575734.677523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575734.677525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575734.677527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575735.125510:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307338681, at 4307338688
00000100:00000010:1.0:1788575735.125525:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000040:0.0:1788575735.125619:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307338681, at 4307338688
00000100:00000010:0.0:1788575735.125626:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00010000:00000001:3.0:1788575735.573500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575735.573507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575735.573513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575735.573516:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575735.573518:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575735.573521:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575735.573524:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575735.573526:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575735.573528:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575735.573530:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575735.573534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575735.573535:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575735.573537:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575735.573538:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575735.573540:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575735.573541:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575735.573543:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575735.573544:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575735.573596:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575735.573599:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575735.573601:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575735.701514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575735.701521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575735.701523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575735.701529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575735.701532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575735.701536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575735.701539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575735.701542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575735.701544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575736.597495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575736.597501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575736.597506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575736.597508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575736.597511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575736.597512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575736.597567:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575736.597570:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575736.597574:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575736.725504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575736.725512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575736.725514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575736.725519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575736.725523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575736.725527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575736.725529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575736.725531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575736.725534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575737.621520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575737.621526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575737.621532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575737.621533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575737.621537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575737.621538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575737.621627:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575737.621631:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575737.621634:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575737.749487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575737.749494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575737.749496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575737.749509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575737.749514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575737.749518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575737.749520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575737.749524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575737.749526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575738.645494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575738.645502:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575738.645504:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575738.645508:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575738.645513:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575738.645515:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575738.645538:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575738.645540:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575738.645545:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575738.645546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575738.645550:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575738.645551:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575738.645615:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575738.645618:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575738.645621:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575738.773512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575738.773530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575738.773532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575738.773537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575738.773541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575738.773544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575738.773546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575738.773549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575738.773552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575739.287624:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575739.287634:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575739.287637:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575739.287641:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575739.287645:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575739.287652:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12670
00000400:00000200:3.0:1788575739.287659:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575739.287665:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575739.287667:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575739.287669:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287671:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575739.287672:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287674:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575739.287677:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575739.287678:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575739.287680:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575739.287682:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575739.287683:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575739.287685:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575739.287686:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287688:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575739.287689:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287691:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575739.287693:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575739.287695:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287697:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575739.287699:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575739.287701:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575739.287702:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575739.287704:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575739.287705:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575739.287713:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000020:00000040:3.0:1788575739.287715:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575739.287717:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575739.287719:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575739.287721:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575739.287723:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575739.287726:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575739.287731:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575739.287736:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575739.287743:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575739.287750:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575739.287753:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575739.287756:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575739.287777:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575739.287780:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575739.287782:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575739.287783:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575739.287784:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575739.287786:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575739.287788:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12670
00000400:00000200:3.0:1788575739.287791:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575739.287792:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575739.287794:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575739.287796:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287797:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575739.287798:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287799:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575739.287801:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575739.287801:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575739.287802:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575739.287803:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575739.287804:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575739.287805:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575739.287807:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287808:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575739.287809:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287810:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575739.287811:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575739.287813:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287814:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575739.287815:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575739.287819:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575739.287820:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575739.287821:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575739.287824:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575739.287825:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575739.287826:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575739.287828:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575739.287830:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575739.287831:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575739.287832:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575739.287833:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575739.287834:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575739.287835:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575739.287837:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575739.287840:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575739.287842:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575739.287847:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575739.287848:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575739.287850:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575739.287854:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575739.287855:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575739.287857:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575739.287858:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575739.287859:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575739.287860:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575739.287861:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12670
00000400:00000200:3.0:1788575739.287863:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575739.287865:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575739.287866:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575739.287867:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287868:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575739.287869:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287870:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575739.287871:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575739.287872:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575739.287873:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575739.287874:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575739.287875:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575739.287877:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575739.287878:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287879:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575739.287880:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287882:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575739.287883:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575739.287884:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575739.287885:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575739.287886:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575739.287887:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575739.287888:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575739.287889:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575739.287890:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575739.287892:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575739.287892:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575739.287894:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575739.287895:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575739.287896:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575739.287897:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575739.287898:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575739.287898:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575739.287899:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575739.287901:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575739.287902:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575739.287904:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575739.287908:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575739.287909:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575739.287911:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575739.287914:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575739.287916:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575739.287919:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575739.287922:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12680)
00000100:00000001:2.0:1788575739.287978:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575739.287982:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575739.287984:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575739.287990:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.287998:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575739.288000:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575739.288004:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575739.288006:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575739.288008:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575739.288010:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393232000:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575739.288013:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575739.288016:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575739.288018:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575739.288020:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575739.288023:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575739.288025:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575739.288029:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575739.288032:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575739.288036:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575739.288040:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393232000, portal 25
00000100:00000001:2.0:1788575739.288042:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575739.288043:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:2.0:1788575739.288047:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288053:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575739.288061:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575739.288063:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575739.288068:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393232000, offset 0
00000400:00000200:2.0:1788575739.288073:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575739.288081:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575739.288088:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575739.288093:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575739.288096:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575739.288100:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575739.288103:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575739.288108:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331068:331068:256:4294967295] 192.168.202.110@tcp LPNI seq info [331025:331025:8:4294967295]
00000400:00000200:2.0:1788575739.288112:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575739.288122:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575739.288128:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575739.288133:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575739.288139:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575739.288143:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575739.288150:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575739.288153:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575739.288158:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575739.288161:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575739.288166:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575739.288171:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575739.288176:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575739.288179:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575739.288183:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575739.288186:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288195:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575739.288198:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288206:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575739.288207:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575739.288212:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575739.288216:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575739.288218:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307343803
00000100:00000010:2.0:1788575739.288223:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cde0.
00000100:00000001:2.0:1788575739.288230:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575739.288244:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307343803
00000100:00000010:2.0:1788575739.288247:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cb40.
00000100:00000001:2.0:1788575739.288250:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575739.288254:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575739.288257:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575739.288259:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575739.288264:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575739/real 1788575739]  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575739.288275:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575739.288277:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575739.288280:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575739.288282:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575739.288284:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575739.288287:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575739.288290:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575739.288299:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288306:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575739.288307:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288310:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575739.288312:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575739.288316:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288324:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575739.288328:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288335:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575739.288338:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288340:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575739.288341:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288343:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575739.288345:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575739.288348:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575739.288351:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575739.288355:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575739.288359:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575739.288368:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393232000:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575739.288373:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575739.288376:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288382:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575739.288383:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575739.288386:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575739.288387:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575739.288389:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575739.288412:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575739.288416:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575739.288418:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575739.288419:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575739.288421:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575739.288425:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575739.288426:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575739.288430:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575739.288432:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575739.288436:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575739.288438:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393232128, portal 10
00000100:00000001:2.0:1788575739.288441:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575739.288442:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:2.0:1788575739.288448:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288454:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575739.288456:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575739.288458:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575739.288461:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393232128, offset 0
00000400:00000200:2.0:1788575739.288465:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575739.288468:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575739.288472:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575739.288475:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575739.288477:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575739.288479:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575739.288482:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575739.288485:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331069:331069:256:4294967295] 192.168.202.110@tcp LPNI seq info [331026:331026:8:4294967295]
00000400:00000200:2.0:1788575739.288503:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575739.288510:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575739.288514:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575739.288518:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575739.288521:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575739.288524:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575739.288527:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575739.288530:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575739.288534:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575739.288535:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575739.288538:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575739.288541:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575739.288544:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575739.288545:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575739.288548:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575739.288551:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288559:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575739.288562:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288567:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575739.288569:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575739.288572:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575739.288576:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575739.288580:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575739.288581:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307342889
00000100:00000001:2.0:1788575739.288584:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307343803
00000100:00000001:2.0:1788575739.288586:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307342889
00000100:00000001:2.0:1788575739.288588:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307343803
00000100:00000001:2.0:1788575739.288590:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575739.288592:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575739.288594:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575739.288597:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575739/real 1788575739]  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575739.288604:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575739.288606:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575739.288607:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575739.288609:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575739.288611:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575739.288613:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575739.288616:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575739.288623:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288627:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575739.288628:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288629:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575739.288630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575739.288632:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288637:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575739.288639:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288644:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575739.288645:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288646:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575739.288647:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288648:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575739.288649:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575739.288651:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575739.288652:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575739.288653:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575739.288655:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575739.288659:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575739.288662:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575739.288664:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288668:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575739.288669:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575739.288670:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575739.288671:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575739.288672:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575739.288674:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575739.288676:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575739.288677:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575739.288678:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575739.288679:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575739.288681:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575739.288682:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575739.288685:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575739.288687:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575739.288689:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575739.288691:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393232256, portal 4
00000100:00000001:2.0:1788575739.288692:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575739.288693:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788575739.288696:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288700:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575739.288701:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575739.288703:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575739.288704:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393232256, offset 0
00000400:00000200:2.0:1788575739.288706:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575739.288709:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575739.288711:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575739.288713:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575739.288714:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575739.288716:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575739.288717:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575739.288719:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331070:331070:256:4294967295] 192.168.202.110@tcp LPNI seq info [331027:331027:8:4294967295]
00000400:00000200:2.0:1788575739.288721:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575739.288725:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575739.288728:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575739.288730:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575739.288732:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575739.288734:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575739.288737:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575739.288738:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575739.288741:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575739.288742:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575739.288744:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575739.288746:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575739.288748:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575739.288749:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575739.288750:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575739.288752:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288757:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575739.288758:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288762:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575739.288763:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575739.288765:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575739.288766:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575739.288768:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575739.288769:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307343035
00000100:00000001:2.0:1788575739.288771:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307343803
00000100:00000001:2.0:1788575739.288772:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307343035
00000100:00000001:2.0:1788575739.288773:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307343803
00000100:00000001:2.0:1788575739.288774:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575739.288775:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575739.288776:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575739.288778:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575739/real 1788575739]  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575739.288782:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575739.288783:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575739.288784:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575739.288785:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575739.288785:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575739.288786:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575739.288788:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575739.288792:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288800:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575739.288801:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575739.288803:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575739.288805:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288810:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575739.288814:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288820:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575739.288822:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288823:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575739.288824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288826:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575739.288827:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575739.288830:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575739.288833:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575739.288835:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575739.288839:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575739.288846:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575739.288850:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288852:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575739.288855:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393232000/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288861:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575739.288862:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575739.288865:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575739.288867:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575739.288869:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575739.288871:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575739.288873:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575739.288875:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575739.288879:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575739.288880:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575739.288882:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:2.0:1788575739.288885:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575739.288886:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288887:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575739.288891:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393232128/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288897:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575739.288899:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575739.288900:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575739.288902:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575739.288903:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575739.288905:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575739.288906:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575739.288908:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575739.288910:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575739.288911:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575739.288912:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575739.288914:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575739.288915:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288916:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575739.288918:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393232256/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575739.288922:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575739.288923:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575739.288924:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575739.288925:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575739.288926:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575739.288927:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575739.288928:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575739.288929:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575739.288930:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575739.288931:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575739.288932:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575739.288933:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575739.288934:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288935:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.288940:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575739.288942:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575739.288943:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575739.288944:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575739.669525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575739.669532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575739.669537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575739.669539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575739.669542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575739.669543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575739.669601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575739.669605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575739.669608:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575739.797483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575739.797490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575739.797492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575739.797497:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575739.797500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575739.797504:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.797506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575739.797508:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575739.797511:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575740.245507:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307343803, at 4307343808
00000100:00000040:1.0:1788575740.245507:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307343803, at 4307343808
00000100:00000010:0.0:1788575740.245523:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000100:00000010:1.0:1788575740.245524:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00010000:00000001:3.0:1788575740.693520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575740.693526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575740.693531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575740.693533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575740.693536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575740.693537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575740.693599:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575740.693603:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575740.693606:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575740.821536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575740.821542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575740.821544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575740.821558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575740.821561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575740.821565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575740.821568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575740.821570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575740.821574:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575741.717493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575741.717499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575741.717504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575741.717505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575741.717509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575741.717510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575741.717610:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575741.717612:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575741.717614:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575741.845507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575741.845513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575741.845515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575741.845521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575741.845525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575741.845530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575741.845533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575741.845536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575741.845540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575742.741505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575742.741511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575742.741517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575742.741518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575742.741522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575742.741523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575742.741586:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575742.741590:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575742.741593:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575742.869511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575742.869524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575742.869526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575742.869534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575742.869537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575742.869541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575742.869543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575742.869546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575742.869549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575743.765478:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575743.765483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575743.765487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575743.765487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575743.765490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575743.765491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575743.765522:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575743.765525:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575743.765528:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575743.893522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575743.893530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575743.893532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575743.893539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575743.893543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575743.893547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575743.893550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575743.893553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575743.893555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575744.405535:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575744.405547:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575744.405550:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575744.405560:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575744.405566:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575744.405573:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12675
00000400:00000200:3.0:1788575744.405583:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575744.405591:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575744.405595:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575744.405598:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405602:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575744.405605:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405607:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575744.405611:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575744.405613:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575744.405615:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575744.405618:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575744.405619:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575744.405621:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575744.405623:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405624:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575744.405626:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405627:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575744.405629:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575744.405631:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405633:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575744.405635:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575744.405637:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575744.405639:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.405640:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575744.405642:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575744.405651:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000020:00000040:3.0:1788575744.405653:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575744.405657:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575744.405658:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.405660:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575744.405663:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575744.405666:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575744.405672:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575744.405677:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575744.405685:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575744.405693:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575744.405696:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575744.405699:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575744.405714:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575744.405717:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575744.405719:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575744.405720:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575744.405721:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575744.405723:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575744.405730:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12675
00000400:00000200:3.0:1788575744.405733:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575744.405735:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575744.405738:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575744.405741:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405743:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575744.405746:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405748:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575744.405750:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575744.405751:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575744.405753:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575744.405754:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575744.405758:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575744.405761:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575744.405764:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575744.405766:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575744.405767:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575744.405769:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575744.405773:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405775:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575744.405777:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575744.405778:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00080000:3.0:1788575744.405782:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405786:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575744.405787:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575744.405789:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575744.405793:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575744.405797:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.405800:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575744.405804:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:3.0:1788575744.405805:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788575744.405806:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788575744.405809:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.405812:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575744.405814:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
02000000:00000001:2.0:1788575744.405815:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1788575744.405816:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575744.405820:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575744.405820:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575744.405823:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575744.405824:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.405827:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:2.0:1788575744.405829:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393232384:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575744.405830:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575744.405834:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575744.405836:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575744.405840:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:3.0:1788575744.405840:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575744.405844:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.405846:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575744.405848:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575744.405855:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
00000100:00100000:3.0:1788575744.405856:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575744.405865:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.405867:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575744.405870:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575744.405873:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575744.405876:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575744.405883:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00000001:3.0:1788575744.405892:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575744.405895:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00080000:3.0:1788575744.405895:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575744.405900:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575744.405901:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575744.405902:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575744.405904:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000200:2.0:1788575744.405905:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393232384, portal 25
00000100:00080000:3.0:1788575744.405908:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12675
00000100:00000001:2.0:1788575744.405911:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000400:00000200:3.0:1788575744.405913:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575744.405915:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575744.405917:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575744.405919:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405920:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575744.405924:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575744.405927:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000001:3.0:1788575744.405927:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575744.405934:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575744.405935:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575744.405937:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575744.405940:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575744.405940:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575744.405945:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575744.405949:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575744.405957:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575744.405959:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575744.405962:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575744.405967:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575744.405968:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405971:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:2.0:1788575744.405972:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000020:00000040:3.0:1788575744.405973:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575744.405978:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575744.405980:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575744.405982:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575744.405984:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575744.405985:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.405986:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:2.0:1788575744.405987:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393232384, offset 0
00000008:00100000:3.0:1788575744.405988:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575744.405992:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.405995:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575744.405999:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000400:00000200:2.0:1788575744.406002:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575744.406003:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575744.406007:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575744.406009:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575744.406010:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575744.406012:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575744.406014:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575744.406015:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575744.406021:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000400:00000200:2.0:1788575744.406025:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788575744.406026:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575744.406033:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:3.0:1788575744.406034:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575744.406039:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575744.406045:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575744.406046:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575744.406049:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575744.406050:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575744.406055:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331071:331071:256:4294967295] 192.168.202.110@tcp LPNI seq info [331028:331028:8:4294967295]
00000100:00000040:3.0:1788575744.406057:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575744.406064:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575744.406071:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575744.406075:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575744.406080:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575744.406086:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1788575744.406089:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12685)
00000800:00000200:2.0:1788575744.406096:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575744.406102:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575744.406110:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575744.406115:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575744.406121:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575744.406124:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575744.406128:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575744.406130:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575744.406136:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575744.406139:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575744.406144:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575744.406145:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575744.406149:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575744.406152:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406159:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575744.406163:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406170:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575744.406172:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575744.406177:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575744.406181:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575744.406184:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307348921
00000100:00000010:2.0:1788575744.406189:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c120.
00000100:00000001:2.0:1788575744.406205:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575744.406217:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307348921
00000100:00000010:2.0:1788575744.406221:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c780.
00000100:00000001:2.0:1788575744.406224:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575744.406227:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575744.406229:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575744.406232:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575744.406237:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575744/real 1788575744]  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575744.406256:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575744.406258:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575744.406261:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575744.406264:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575744.406265:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575744.406267:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575744.406270:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575744.406275:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406279:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575744.406280:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406282:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575744.406284:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575744.406287:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406293:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575744.406298:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406304:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575744.406315:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406317:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575744.406318:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406321:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575744.406323:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575744.406326:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575744.406330:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575744.406332:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575744.406336:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575744.406342:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393232384:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575744.406347:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406349:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575744.406351:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393232384/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406355:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575744.406357:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575744.406358:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575744.406360:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575744.406362:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575744.406362:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575744.406364:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575744.406366:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575744.406369:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575744.406370:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575744.406371:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575744.406373:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575744.406374:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406375:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406381:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575744.406383:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575744.406385:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575744.406386:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575744.406388:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575744.406428:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406434:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575744.406435:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575744.406437:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575744.406438:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575744.406439:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575744.406442:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232512:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575744.406444:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575744.406446:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575744.406447:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575744.406449:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575744.406453:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575744.406455:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575744.406459:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575744.406462:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575744.406465:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575744.406468:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393232512, portal 10
00000100:00000001:2.0:1788575744.406470:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575744.406471:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788575744.406478:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406484:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575744.406487:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575744.406489:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575744.406491:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393232512, offset 0
00000400:00000200:2.0:1788575744.406495:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575744.406499:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575744.406503:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575744.406506:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575744.406508:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575744.406510:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575744.406513:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575744.406516:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331072:331072:256:4294967295] 192.168.202.110@tcp LPNI seq info [331029:331029:8:4294967295]
00000400:00000200:2.0:1788575744.406520:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575744.406526:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575744.406531:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575744.406534:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575744.406538:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575744.406541:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575744.406545:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575744.406554:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575744.406559:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575744.406561:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575744.406564:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575744.406567:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575744.406570:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575744.406572:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575744.406574:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575744.406578:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406585:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575744.406588:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406595:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575744.406596:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575744.406600:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575744.406604:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575744.406606:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575744.406607:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307348007
00000100:00000001:2.0:1788575744.406609:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307348921
00000100:00000001:2.0:1788575744.406610:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307348007
00000100:00000001:2.0:1788575744.406612:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307348921
00000100:00000001:2.0:1788575744.406613:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575744.406614:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575744.406615:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575744.406618:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575744/real 1788575744]  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575744.406622:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575744.406623:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575744.406624:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575744.406626:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575744.406627:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575744.406628:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575744.406630:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575744.406635:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406639:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575744.406640:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406641:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575744.406642:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575744.406644:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406647:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575744.406649:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406653:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575744.406655:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406656:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575744.406657:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406658:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575744.406659:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575744.406661:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575744.406663:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575744.406664:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575744.406666:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575744.406670:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232512:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575744.406672:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575744.406674:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406682:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575744.406683:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575744.406685:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575744.406686:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575744.406688:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575744.406690:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232640:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575744.406693:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575744.406695:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575744.406697:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575744.406699:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575744.406702:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575744.406703:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575744.406707:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575744.406709:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575744.406712:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575744.406715:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393232640, portal 4
00000100:00000001:2.0:1788575744.406717:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575744.406718:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575744.406725:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406730:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575744.406732:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575744.406734:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575744.406737:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393232640, offset 0
00000400:00000200:2.0:1788575744.406740:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575744.406744:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575744.406747:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575744.406750:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575744.406753:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575744.406755:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575744.406757:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575744.406761:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331073:331073:256:4294967295] 192.168.202.110@tcp LPNI seq info [331030:331030:8:4294967295]
00000400:00000200:2.0:1788575744.406766:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575744.406772:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575744.406776:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575744.406780:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575744.406784:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575744.406786:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575744.406789:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575744.406791:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575744.406793:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575744.406795:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575744.406797:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575744.406799:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575744.406801:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575744.406802:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575744.406804:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575744.406806:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406811:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575744.406813:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406817:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575744.406818:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575744.406820:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575744.406822:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575744.406824:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575744.406825:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307348153
00000100:00000001:2.0:1788575744.406827:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307348921
00000100:00000001:2.0:1788575744.406828:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307348153
00000100:00000001:2.0:1788575744.406829:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307348921
00000100:00000001:2.0:1788575744.406830:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575744.406832:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575744.406833:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575744.406835:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575744/real 1788575744]  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575744.406840:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575744.406841:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575744.406842:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575744.406843:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575744.406844:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575744.406846:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575744.406848:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575744.406856:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406864:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575744.406865:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406867:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575744.406868:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575744.406871:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406878:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575744.406882:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406889:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575744.406891:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406893:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575744.406894:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406896:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575744.406897:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575744.406900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575744.406902:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575744.406903:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575744.406906:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575744.406911:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232640:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575744.406913:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406914:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575744.406916:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393232512/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406919:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575744.406920:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575744.406922:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575744.406924:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575744.406925:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575744.406925:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575744.406927:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575744.406928:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575744.406929:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575744.406930:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575744.406931:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575744.406932:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575744.406933:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406934:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575744.406936:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393232640/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575744.406939:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575744.406940:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575744.406941:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575744.406942:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575744.406942:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575744.406943:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575744.406944:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575744.406945:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:2.0:1788575744.406946:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575744.406947:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575744.406948:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575744.406949:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575744.406950:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406951:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.406953:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575744.406954:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575744.406955:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575744.406956:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575744.789495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575744.789499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575744.789502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575744.789505:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575744.789507:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575744.789509:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575744.789512:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575744.789513:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575744.789515:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575744.789523:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575744.789527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575744.789529:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575744.789530:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575744.789531:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575744.789533:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575744.789534:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575744.789535:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575744.789537:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575744.789596:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575744.789598:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575744.789603:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575744.917486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575744.917493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575744.917496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575744.917501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575744.917505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575744.917509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.917512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575744.917514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575744.917517:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575745.365458:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307348921, at 4307348928
00000100:00000040:0.0:1788575745.365464:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307348921, at 4307348928
00000100:00000010:1.0:1788575745.365473:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000100:00000010:0.0:1788575745.365475:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00010000:00000001:3.0:1788575745.813505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575745.813512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575745.813517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575745.813518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575745.813521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575745.813523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575745.813573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575745.813576:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575745.813578:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575745.941938:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575745.941945:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575745.941947:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575745.941954:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575745.941957:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575745.941961:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575745.941964:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575745.941966:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575745.941969:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575746.837517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575746.837523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575746.837529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575746.837530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575746.837533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575746.837535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575746.837597:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575746.837602:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575746.837605:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575746.965502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575746.965509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575746.965511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575746.965522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575746.965527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575746.965529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575746.965534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575746.965536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575746.965538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575747.861473:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575747.861481:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575747.861483:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575747.861486:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575747.861491:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575747.861493:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575747.861494:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575747.861497:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575747.861500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575747.861501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575747.861505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575747.861506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575747.861559:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575747.861562:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575747.861565:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575747.989539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575747.989547:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575747.989549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575747.989555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575747.989558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575747.989561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575747.989564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575747.989566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575747.989569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575748.885489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575748.885494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575748.885498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575748.885499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575748.885501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575748.885502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575748.885564:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575748.885567:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575748.885571:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575749.013447:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575749.013453:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575749.013454:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575749.013458:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575749.013460:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575749.013463:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.013465:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575749.013466:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.013468:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575749.525483:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575749.525491:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575749.525494:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575749.525497:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575749.525501:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575749.525507:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12680
00000400:00000200:1.0:1788575749.525516:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575749.525522:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575749.525525:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575749.525529:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525532:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575749.525534:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525537:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575749.525541:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575749.525542:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575749.525544:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575749.525547:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575749.525548:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575749.525550:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575749.525551:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525552:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575749.525553:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525555:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575749.525557:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575749.525559:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525561:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575749.525563:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575749.525566:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575749.525568:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575749.525569:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575749.525571:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575749.525578:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788575749.525581:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575749.525584:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575749.525585:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575749.525587:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575749.525589:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575749.525592:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575749.525598:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788575749.525604:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575749.525613:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575749.525623:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575749.525626:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575749.525633:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575749.525653:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575749.525655:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575749.525658:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575749.525659:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575749.525660:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575749.525662:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575749.525664:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12680
00000400:00000200:1.0:1788575749.525666:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575749.525668:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575749.525669:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575749.525671:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525673:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575749.525674:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525675:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575749.525677:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575749.525677:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575749.525678:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575749.525680:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575749.525681:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575749.525682:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575749.525683:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525685:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575749.525686:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525687:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575749.525688:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575749.525690:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525691:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575749.525693:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575749.525697:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575749.525698:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575749.525700:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575749.525704:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788575749.525704:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00000001:1.0:1788575749.525706:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575749.525709:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575749.525710:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000010:1.0:1788575749.525713:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000100:00000001:2.0:1788575749.525713:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1788575749.525717:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575749.525719:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575749.525719:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575749.525721:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575749.525722:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575749.525724:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575749.525729:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000040:2.0:1788575749.525733:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575749.525734:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575749.525743:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575749.525755:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575749.525759:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575749.525760:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575749.525764:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575749.525765:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575749.525769:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1788575749.525770:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575749.525771:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575749.525778:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393232768:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575749.525785:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575749.525789:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1788575749.525791:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575749.525792:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575749.525797:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:1.0:1788575749.525798:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000010:2.0:1788575749.525802:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
00000100:00080000:1.0:1788575749.525803:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575749.525807:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:2.0:1788575749.525807:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575749.525810:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575749.525814:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575749.525816:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:2.0:1788575749.525818:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00080000:1.0:1788575749.525822:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12680
00000400:00000200:1.0:1788575749.525826:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575749.525829:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575749.525832:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000010:2.0:1788575749.525834:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000040:1.0:1788575749.525836:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525839:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575749.525841:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525843:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:2.0:1788575749.525844:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393232768, portal 25
00000020:00000001:1.0:1788575749.525847:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575749.525849:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575749.525851:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575749.525853:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000020:00000040:1.0:1788575749.525855:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:2.0:1788575749.525858:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345470336 : -130483364081280 : ffff8953796f5180)
00000020:00000001:1.0:1788575749.525859:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575749.525863:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575749.525865:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575749.525870:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525876:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575749.525877:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1788575749.525882:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575749.525883:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575749.525886:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000001:1.0:1788575749.525887:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:2.0:1788575749.525890:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393232768, offset 0
00000020:00000040:1.0:1788575749.525892:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575749.525896:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788575749.525899:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525903:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575749.525906:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575749.525907:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575749.525911:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575749.525913:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:1.0:1788575749.525917:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575749.525920:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1788575749.525921:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788575749.525924:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00100000:1.0:1788575749.525925:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575749.525928:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00000001:1.0:1788575749.525931:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575749.525933:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575749.525937:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000010:1.0:1788575749.525939:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000400:00000200:2.0:1788575749.525941:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331074:331074:256:4294967295] 192.168.202.110@tcp LPNI seq info [331031:331031:8:4294967295]
00000020:00000040:1.0:1788575749.525945:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575749.525947:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575749.525952:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575749.525954:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575749.525957:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575749.525960:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575749.525961:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1788575749.525963:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575749.525970:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000800:00000200:2.0:1788575749.525970:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575749.525976:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575749.525977:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000100:00100000:1.0:1788575749.525983:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575749.526007:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575749.526009:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575749.526014:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788575749.526015:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575749.526023:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1788575749.526028:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575749.526033:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.1:1788575749.526033:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575749.526040:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:1.0:1788575749.526041:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000100:2.0:1788575749.526048:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788575749.526049:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12690)
00000800:00000010:2.0:1788575749.526053:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575749.526061:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575749.526067:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575749.526073:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575749.526075:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575749.526079:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575749.526084:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526094:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575749.526098:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526105:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575749.526106:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575749.526112:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575749.526116:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575749.526119:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307354041
00000100:00000010:2.0:1788575749.526124:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c5a0.
00000100:00000001:2.0:1788575749.526130:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575749.526144:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307354041
00000100:00000010:2.0:1788575749.526148:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c060.
00000100:00000001:2.0:1788575749.526152:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575749.526155:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575749.526158:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575749.526160:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575749.526166:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575749/real 1788575749]  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575749.526177:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575749.526179:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575749.526182:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575749.526184:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575749.526186:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575749.526188:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575749.526192:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575749.526205:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526212:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575749.526214:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526216:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575749.526218:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575749.526222:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526228:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575749.526232:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526239:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575749.526242:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526243:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575749.526245:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526247:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575749.526249:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575749.526251:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575749.526255:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575749.526257:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575749.526261:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575749.526278:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393232768:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575749.526283:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526285:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575749.526289:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5180 x1875444393232768/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526301:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575749.526303:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575749.526305:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575749.526308:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575749.526310:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575749.526312:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575749.526314:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575749.526316:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:2.0:1788575749.526319:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575749.526321:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575749.526323:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5180.
00000100:00000001:2.0:1788575749.526326:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575749.526327:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526329:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526336:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575749.526338:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575749.526339:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575749.526341:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575749.526342:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575749.526346:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526352:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575749.526353:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575749.526355:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575749.526357:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575749.526359:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575749.526361:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232896:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575749.526364:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575749.526367:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575749.526368:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575749.526370:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575749.526373:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575749.526375:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575749.526378:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575749.526382:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575749.526385:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575749.526388:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393232896, portal 10
00000100:00000001:2.0:1788575749.526390:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575749.526417:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345476608 : -130483364075008 : ffff8953796f6a00)
00000100:00000040:2.0:1788575749.526425:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526431:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575749.526433:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575749.526436:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575749.526438:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393232896, offset 0
00000400:00000200:2.0:1788575749.526442:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575749.526446:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575749.526458:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575749.526461:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575749.526464:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575749.526466:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575749.526468:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575749.526471:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331075:331075:256:4294967295] 192.168.202.110@tcp LPNI seq info [331032:331032:8:4294967295]
00000400:00000200:2.0:1788575749.526475:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575749.526481:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575749.526485:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575749.526489:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575749.526492:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575749.526495:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575749.526499:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575749.526502:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575749.526507:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575749.526509:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575749.526511:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575749.526515:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575749.526518:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575749.526519:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575749.526522:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575749.526526:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526533:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575749.526536:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526542:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575749.526544:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575749.526547:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575749.526550:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575749.526553:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575749.526555:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307353127
00000100:00000001:2.0:1788575749.526558:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307354041
00000100:00000001:2.0:1788575749.526560:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307353127
00000100:00000001:2.0:1788575749.526561:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307354041
00000100:00000001:2.0:1788575749.526562:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575749.526564:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575749.526565:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575749.526568:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575749/real 1788575749]  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575749.526572:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575749.526574:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575749.526575:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575749.526576:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575749.526578:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575749.526579:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575749.526581:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575749.526585:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526593:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575749.526594:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526595:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575749.526596:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575749.526598:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526601:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575749.526604:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526608:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575749.526609:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526610:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575749.526611:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526612:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575749.526613:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575749.526615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575749.526616:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575749.526618:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575749.526620:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575749.526624:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393232896:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575749.526626:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575749.526628:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526632:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575749.526633:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575749.526634:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575749.526636:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575749.526637:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575749.526639:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575749.526641:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575749.526642:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575749.526643:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575749.526644:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575749.526646:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575749.526647:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575749.526649:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575749.526651:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575749.526652:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575749.526654:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393233024, portal 4
00000100:00000001:2.0:1788575749.526655:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575749.526656:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345468544 : -130483364083072 : ffff8953796f4a80)
00000100:00000040:2.0:1788575749.526660:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526664:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575749.526665:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575749.526666:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575749.526668:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393233024, offset 0
00000400:00000200:2.0:1788575749.526670:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575749.526672:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575749.526674:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575749.526676:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575749.526677:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575749.526679:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575749.526680:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575749.526682:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331076:331076:256:4294967295] 192.168.202.110@tcp LPNI seq info [331033:331033:8:4294967295]
00000400:00000200:2.0:1788575749.526685:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575749.526689:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575749.526692:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575749.526695:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575749.526699:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575749.526701:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575749.526705:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575749.526708:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575749.526711:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575749.526713:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575749.526716:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575749.526719:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575749.526722:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575749.526723:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575749.526726:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575749.526729:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526735:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575749.526738:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526743:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575749.526745:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575749.526748:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575749.526751:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575749.526754:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575749.526756:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307353273
00000100:00000001:2.0:1788575749.526758:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307354041
00000100:00000001:2.0:1788575749.526760:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307353273
00000100:00000001:2.0:1788575749.526762:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307354041
00000100:00000001:2.0:1788575749.526764:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575749.526766:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575749.526768:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575749.526771:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575749/real 1788575749]  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575749.526779:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575749.526781:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575749.526783:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575749.526786:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575749.526787:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575749.526790:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575749.526794:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575749.526802:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526808:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575749.526809:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526811:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575749.526813:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575749.526816:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526822:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575749.526826:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526832:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575749.526834:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526836:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575749.526837:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526839:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575749.526841:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575749.526844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575749.526848:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575749.526850:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575749.526855:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575749.526863:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575749.526867:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526869:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575749.526872:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6a00 x1875444393232896/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526878:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575749.526879:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575749.526881:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575749.526884:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575749.526885:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575749.526887:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575749.526889:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575749.526890:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:2.0:1788575749.526892:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575749.526894:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575749.526895:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6a00.
00000100:00000001:2.0:1788575749.526897:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575749.526898:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526900:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575749.526903:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4a80 x1875444393233024/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575749.526909:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575749.526910:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575749.526911:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575749.526913:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575749.526914:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575749.526915:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575749.526916:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575749.526917:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:2.0:1788575749.526918:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575749.526919:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575749.526920:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4a80.
00000100:00000001:2.0:1788575749.526922:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575749.526922:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526924:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575749.526926:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575749.526927:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575749.526928:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575749.526929:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575749.909513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575749.909519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575749.909523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575749.909525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575749.909528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575749.909530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575749.909582:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575749.909586:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575749.909590:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575750.037491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575750.037497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575750.037499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575750.037505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575750.037508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575750.037512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575750.037514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575750.037517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575750.037519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575750.485527:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307354041, at 4307354048
00000100:00000040:1.0:1788575750.485527:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307354041, at 4307354048
00000100:00000010:1.0:1788575750.485544:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000100:00000010:0.0:1788575750.485545:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000800:00000200:1.0:1788575750.549533:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575750.549543:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:1.0:1788575750.549547:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575750.549706:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575750.549726:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575750.549732:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575750.549739:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575750.549743:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575750.933502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575750.933508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575750.933514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575750.933515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575750.933519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575750.933520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575750.933582:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575750.933584:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575750.933587:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575751.061510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575751.061517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575751.061519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575751.061524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575751.061528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575751.061532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575751.061535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575751.061537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575751.061540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575751.957500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575751.957506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575751.957510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575751.957511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575751.957513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575751.957514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575751.957569:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575751.957572:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575751.957575:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575752.085501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575752.085508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575752.085511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575752.085519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575752.085523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575752.085526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575752.085529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575752.085532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575752.085535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575752.981479:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575752.981485:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575752.981488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575752.981489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575752.981491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575752.981492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575752.981544:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575752.981548:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575752.981551:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575753.109527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575753.109537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575753.109540:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575753.109545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575753.109548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575753.109552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575753.109554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575753.109568:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575753.109571:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575754.005506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575754.005512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575754.005517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575754.005519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575754.005523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575754.005524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575754.005592:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575754.005596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575754.005599:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575754.133498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575754.133503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575754.133505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575754.133515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575754.133518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575754.133521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.133523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575754.133525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.133527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575754.645526:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575754.645537:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575754.645539:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575754.645542:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575754.645546:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575754.645551:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12685
00000400:00000200:1.0:1788575754.645559:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575754.645564:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575754.645567:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575754.645570:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575754.645573:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575754.645575:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575754.645578:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575754.645582:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575754.645584:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575754.645587:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575754.645591:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575754.645593:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575754.645595:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575754.645598:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575754.645600:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575754.645603:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575754.645604:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575754.645606:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575754.645608:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575754.645610:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575754.645612:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575754.645615:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575754.645617:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575754.645619:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575754.645621:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575754.645628:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788575754.645631:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575754.645634:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575754.645635:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575754.645637:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575754.645639:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575754.645642:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575754.645648:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788575754.645653:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575754.645660:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575754.645667:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575754.645671:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575754.645675:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575754.645695:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575754.645699:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575754.645702:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575754.645703:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575754.645705:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575754.645707:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575754.645710:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12685
00000400:00000200:1.0:1788575754.645713:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575754.645716:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575754.645719:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575754.645722:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575754.645724:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575754.645726:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575754.645728:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575754.645730:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575754.645732:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575754.645734:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575754.645736:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575754.645750:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575754.645755:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575754.645758:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1788575754.645765:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575754.645765:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788575754.645770:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575754.645778:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575754.645780:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575754.645782:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575754.645784:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575754.645787:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575754.645790:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788575754.645791:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575754.645794:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1788575754.645795:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575754.645799:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575754.645802:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:2.0:1788575754.645806:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393233152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788575754.645807:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575754.645812:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575754.645814:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575754.645818:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1788575754.645819:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575754.645821:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575754.645823:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575754.645826:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788575754.645827:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575754.645833:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1788575754.645833:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
00000008:00000001:1.0:1788575754.645839:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575754.645839:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575754.645843:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575754.645846:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:1.0:1788575754.645848:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000400:00000010:2.0:1788575754.645851:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000020:00000040:1.0:1788575754.645853:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575754.645858:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575754.645859:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575754.645860:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000001:1.0:1788575754.645861:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575754.645865:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575754.645867:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575754.645869:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393233152, portal 25
02000000:00000010:1.0:1788575754.645872:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:2.0:1788575754.645874:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575754.645877:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345466752 : -130483364084864 : ffff8953796f4380)
00000100:00000001:1.0:1788575754.645878:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575754.645884:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1788575754.645887:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575754.645899:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1788575754.645903:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1788575754.645905:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575754.645908:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575754.645910:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000040:1.0:1788575754.645915:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575754.645916:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393233152, offset 0
00000100:00000001:1.0:1788575754.645928:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575754.645928:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788575754.645933:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575754.645936:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:2.0:1788575754.645937:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575754.645938:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575754.645942:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575754.645946:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575754.645947:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:1.0:1788575754.645951:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12685
00000400:00000200:2.0:1788575754.645956:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788575754.645959:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575754.645964:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1788575754.645965:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575754.645969:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575754.645971:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:1.0:1788575754.645975:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575754.645978:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575754.645982:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331077:331077:256:4294967295] 192.168.202.110@tcp LPNI seq info [331034:331034:8:4294967295]
00000100:00000001:1.0:1788575754.645985:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575754.645990:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575754.645991:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575754.645997:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575754.646000:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575754.646001:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575754.646005:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575754.646008:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:2.0:1788575754.646009:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000001:1.0:1788575754.646013:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575754.646017:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.0:1788575754.646021:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788575754.646022:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575754.646026:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575754.646027:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575754.646029:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000010:2.0:1788575754.646029:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000020:00000040:1.0:1788575754.646030:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575754.646035:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575754.646037:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575754.646039:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:2.0:1788575754.646042:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1788575754.646043:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575754.646044:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575754.646046:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575754.646047:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575754.646048:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575754.646054:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:1.0:1788575754.646059:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575754.646061:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:1.0:1788575754.646065:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000800:00000200:2.1:1788575754.646068:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000040:1.0:1788575754.646069:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575754.646075:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575754.646078:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1788575754.646079:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575754.646081:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:2.0:1788575754.646083:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
02000000:00000001:1.0:1788575754.646085:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575754.646089:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575754.646093:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:1.0:1788575754.646094:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788575754.646100:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575754.646101:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00100000:1.0:1788575754.646110:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575754.646111:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575754.646116:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575754.646122:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:1.0:1788575754.646129:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1788575754.646130:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575754.646134:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575754.646143:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646145:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575754.646153:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575754.646156:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575754.646162:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575754.646166:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575754.646168:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575754.646169:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575754.646176:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575754.646177:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12695)
00000400:00000200:2.0:1788575754.646183:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575754.646188:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307359161
00000100:00000010:2.0:1788575754.646194:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cea0.
00000100:00000001:2.0:1788575754.646201:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575754.646216:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307359161
00000100:00000010:2.0:1788575754.646220:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c180.
00000100:00000001:2.0:1788575754.646223:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575754.646227:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575754.646229:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575754.646231:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575754.646236:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575754/real 1788575754]  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575754.646253:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575754.646256:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575754.646258:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575754.646261:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575754.646263:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575754.646266:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575754.646270:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575754.646278:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646286:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575754.646288:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646295:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575754.646298:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575754.646302:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646308:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575754.646312:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646319:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575754.646322:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646323:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575754.646325:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646327:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575754.646329:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575754.646332:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575754.646336:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575754.646338:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575754.646343:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575754.646351:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393233152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575754.646355:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646357:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575754.646361:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4380 x1875444393233152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646373:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575754.646376:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575754.646378:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575754.646381:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575754.646383:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575754.646385:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575754.646387:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575754.646389:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:2.0:1788575754.646416:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575754.646418:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575754.646420:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4380.
00000100:00000001:2.0:1788575754.646423:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575754.646424:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646426:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646433:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575754.646435:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575754.646437:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575754.646439:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575754.646440:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575754.646444:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646452:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575754.646453:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575754.646455:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575754.646457:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575754.646459:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575754.646462:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575754.646465:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575754.646480:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575754.646482:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575754.646484:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575754.646487:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575754.646490:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575754.646493:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575754.646496:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575754.646500:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575754.646503:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393233280, portal 10
00000100:00000001:2.0:1788575754.646504:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575754.646505:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345465856 : -130483364085760 : ffff8953796f4000)
00000100:00000040:2.0:1788575754.646511:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646518:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575754.646520:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575754.646522:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575754.646524:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393233280, offset 0
00000400:00000200:2.0:1788575754.646527:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575754.646531:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575754.646535:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575754.646538:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575754.646541:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575754.646543:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575754.646545:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575754.646548:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331078:331078:256:4294967295] 192.168.202.110@tcp LPNI seq info [331035:331035:8:4294967295]
00000400:00000200:2.0:1788575754.646551:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575754.646558:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575754.646562:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575754.646565:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575754.646569:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575754.646571:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575754.646575:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575754.646577:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575754.646581:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575754.646582:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575754.646585:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575754.646588:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575754.646591:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575754.646593:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575754.646596:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575754.646599:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646606:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575754.646610:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646615:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575754.646617:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575754.646619:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575754.646623:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575754.646626:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575754.646628:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307358247
00000100:00000001:2.0:1788575754.646630:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307359161
00000100:00000001:2.0:1788575754.646632:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307358247
00000100:00000001:2.0:1788575754.646634:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307359161
00000100:00000001:2.0:1788575754.646636:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575754.646638:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575754.646640:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575754.646643:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575754/real 1788575754]  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575754.646650:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575754.646651:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575754.646652:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575754.646655:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575754.646656:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575754.646658:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575754.646661:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575754.646668:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646679:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575754.646681:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646682:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575754.646684:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575754.646687:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646694:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575754.646698:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646704:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575754.646707:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646708:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575754.646710:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646711:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575754.646714:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575754.646716:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575754.646719:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575754.646721:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575754.646725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575754.646732:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575754.646736:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575754.646751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646758:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575754.646760:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575754.646762:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575754.646764:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575754.646766:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575754.646769:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233408:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575754.646772:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575754.646773:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575754.646775:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575754.646776:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575754.646780:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575754.646782:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575754.646785:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575754.646787:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575754.646791:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575754.646794:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393233408, portal 4
00000100:00000001:2.0:1788575754.646796:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575754.646797:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345473920 : -130483364077696 : ffff8953796f5f80)
00000100:00000040:2.0:1788575754.646803:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646808:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575754.646810:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575754.646812:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575754.646814:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393233408, offset 0
00000400:00000200:2.0:1788575754.646818:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575754.646822:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575754.646825:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575754.646828:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575754.646830:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575754.646832:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575754.646834:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575754.646837:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331079:331079:256:4294967295] 192.168.202.110@tcp LPNI seq info [331036:331036:8:4294967295]
00000400:00000200:2.0:1788575754.646841:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575754.646846:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575754.646850:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575754.646853:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575754.646857:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575754.646859:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575754.646863:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575754.646866:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575754.646869:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575754.646870:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575754.646874:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575754.646876:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575754.646879:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575754.646880:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575754.646883:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575754.646886:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646893:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575754.646897:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646902:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575754.646904:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575754.646906:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575754.646909:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575754.646912:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575754.646913:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307358393
00000100:00000001:2.0:1788575754.646915:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307359161
00000100:00000001:2.0:1788575754.646918:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307358393
00000100:00000001:2.0:1788575754.646919:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307359161
00000100:00000001:2.0:1788575754.646922:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575754.646923:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575754.646925:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575754.646929:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575754/real 1788575754]  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575754.646936:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575754.646937:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575754.646940:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575754.646942:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575754.646943:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575754.646945:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575754.646948:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575754.646955:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646960:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575754.646961:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646964:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575754.646965:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575754.646968:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646974:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575754.646977:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.646982:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575754.646985:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646986:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575754.646987:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.646989:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575754.646990:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575754.646992:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575754.646995:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575754.646997:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575754.647001:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575754.647008:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233408:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575754.647011:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.647013:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575754.647016:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4000 x1875444393233280/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.647022:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575754.647024:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575754.647026:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575754.647028:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575754.647029:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575754.647031:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575754.647033:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575754.647034:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:2.0:1788575754.647036:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575754.647038:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575754.647039:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4000.
00000100:00000001:2.0:1788575754.647041:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575754.647042:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.647044:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575754.647047:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5f80 x1875444393233408/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575754.647053:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575754.647054:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575754.647055:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575754.647057:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575754.647059:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575754.647060:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575754.647061:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575754.647063:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:2.0:1788575754.647065:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575754.647066:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575754.647068:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5f80.
00000100:00000001:2.0:1788575754.647070:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575754.647071:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.647074:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575754.647077:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575754.647078:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575754.647080:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575754.647082:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575755.029508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575755.029519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575755.029525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575755.029528:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575755.029530:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575755.029534:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575755.029537:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575755.029539:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575755.029541:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575755.029543:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575755.029547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575755.029548:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575755.029549:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575755.029551:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575755.029552:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575755.029554:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575755.029555:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575755.029557:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575755.029611:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575755.029614:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575755.029616:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575755.157494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575755.157501:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575755.157503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575755.157507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575755.157509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575755.157512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575755.157514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575755.157516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575755.157518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575755.605470:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307359161, at 4307359168
00000100:00000040:0.0:1788575755.605471:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307359161, at 4307359168
00000100:00000010:1.0:1788575755.605485:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000100:00000010:0.0:1788575755.605486:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00010000:00000001:3.0:1788575756.053514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575756.053522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575756.053528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575756.053529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575756.053532:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575756.053533:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575756.053598:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575756.053602:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575756.053605:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575756.181519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575756.181526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575756.181528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575756.181533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575756.181536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575756.181539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575756.181541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575756.181543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575756.181546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575757.077517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575757.077524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575757.077530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575757.077532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575757.077535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575757.077536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575757.077593:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575757.077597:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575757.077601:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575757.205516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575757.205522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575757.205524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575757.205530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575757.205534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575757.205537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575757.205540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575757.205542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575757.205545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575758.101506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575758.101513:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575758.101516:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575758.101519:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575758.101524:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575758.101527:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575758.101528:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575758.101531:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575758.101535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575758.101536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575758.101539:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575758.101540:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575758.101618:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575758.101622:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575758.101625:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575758.229481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575758.229488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575758.229490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575758.229496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575758.229507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575758.229511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575758.229514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575758.229517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575758.229520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575759.125481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575759.125487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575759.125490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575759.125491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575759.125493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575759.125494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575759.125564:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575759.125568:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575759.125571:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575759.253485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575759.253492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575759.253494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575759.253503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575759.253506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575759.253509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.253511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575759.253523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.253527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575759.765498:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575759.765509:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575759.765511:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575759.765516:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575759.765520:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575759.765528:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12690
00000400:00000200:1.0:1788575759.765536:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575759.765542:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575759.765546:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575759.765550:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765554:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575759.765556:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765559:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575759.765563:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575759.765565:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575759.765568:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575759.765571:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575759.765573:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575759.765575:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575759.765578:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765580:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575759.765582:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765584:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575759.765587:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575759.765590:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765593:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575759.765595:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575759.765599:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575759.765602:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575759.765604:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575759.765606:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575759.765615:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788575759.765618:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575759.765622:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575759.765623:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575759.765626:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575759.765629:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575759.765632:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575759.765640:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788575759.765646:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575759.765653:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575759.765668:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575759.765672:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575759.765676:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575759.765697:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575759.765701:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575759.765704:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575759.765706:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575759.765707:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575759.765710:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575759.765712:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12690
00000400:00000200:1.0:1788575759.765716:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575759.765719:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575759.765722:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575759.765724:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788575759.765726:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575759.765728:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575759.765730:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575759.765732:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788575759.765735:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765740:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:2.0:1788575759.765740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788575759.765744:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575759.765747:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575759.765751:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575759.765752:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1788575759.765754:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575759.765755:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575759.765759:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575759.765761:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575759.765763:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575759.765765:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393233536:192.168.202.110@tcp:250:kworker.0
00000020:00000001:1.0:1788575759.765769:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575759.765770:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575759.765774:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:1.0:1788575759.765777:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575759.765778:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575759.765782:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765785:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575759.765787:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765789:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575759.765792:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:1.0:1788575759.765793:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575759.765798:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000010:2.0:1788575759.765799:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
00000100:00000001:1.0:1788575759.765801:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575759.765803:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575759.765805:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575759.765811:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575759.765812:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575759.765816:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575759.765818:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00000001:1.0:1788575759.765819:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575759.765824:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000010:2.0:1788575759.765827:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000008:00000001:1.0:1788575759.765829:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575759.765833:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575759.765837:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000100:00000200:2.0:1788575759.765837:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393233536, portal 25
00000100:00000001:2.0:1788575759.765841:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000020:00000040:1.0:1788575759.765843:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575759.765844:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
02000000:00000001:1.0:1788575759.765848:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575759.765850:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575759.765854:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:2.0:1788575759.765854:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788575759.765857:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575759.765859:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.765868:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575759.765871:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000010:1.0:1788575759.765873:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000400:00000010:2.0:1788575759.765876:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000001:1.0:1788575759.765878:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575759.765881:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393233536, offset 0
00000100:00100000:1.0:1788575759.765887:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575759.765891:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765899:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575759.765902:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575759.765905:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575759.765910:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:1.0:1788575759.765912:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575759.765915:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575759.765921:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575759.765926:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575759.765930:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765931:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575759.765934:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331080:331080:256:4294967295] 192.168.202.110@tcp LPNI seq info [331037:331037:8:4294967295]
00000100:00080000:1.0:1788575759.765936:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575759.765941:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788575759.765944:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575759.765946:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575759.765950:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575759.765953:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1788575759.765955:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:2.0:1788575759.765959:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788575759.765961:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12690
00000800:00000010:2.0:1788575759.765964:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000400:00000200:1.0:1788575759.765969:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:2.0:1788575759.765972:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:1.0:1788575759.765975:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:2.0:1788575759.765978:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000100:00080000:1.0:1788575759.765979:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575759.765985:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575759.765990:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788575759.765990:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788575759.765996:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575759.765996:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000100:00000001:1.0:1788575759.765999:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575759.766003:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000100:2.0:1788575759.766005:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:1.0:1788575759.766006:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000010:2.0:1788575759.766009:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000020:00000001:1.0:1788575759.766010:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575759.766013:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575759.766017:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575759.766019:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575759.766019:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:1.0:1788575759.766026:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575759.766028:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1788575759.766031:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575759.766034:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575759.766041:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575759.766043:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575759.766047:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:1.0:1788575759.766048:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:2.0:1788575759.766052:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788575759.766054:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575759.766061:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575759.766066:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766066:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:1.0:1788575759.766071:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575759.766071:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788575759.766075:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575759.766080:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575759.766084:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575759.766086:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1788575759.766087:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575759.766090:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000008:00000001:1.0:1788575759.766092:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575759.766095:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766098:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575759.766099:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788575759.766104:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575759.766107:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575759.766109:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575759.766112:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766112:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307364281
00000100:00000001:1.0:1788575759.766117:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575759.766119:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575759.766121:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575759.766121:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c420.
02000000:00000010:1.0:1788575759.766126:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:2.0:1788575759.766129:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788575759.766131:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766134:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307364281
00000100:00100000:1.0:1788575759.766138:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:2.0:1788575759.766140:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c9c0.
00000100:00000001:2.0:1788575759.766144:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1788575759.766149:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575759.766149:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1788575759.766153:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766154:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766157:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:1.0:1788575759.766158:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575759.766165:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575759.766171:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575759/real 1788575759]  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575759.766175:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575759.766180:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575759.766186:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12701)
00000400:00000001:2.0:1788575759.766190:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575759.766193:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575759.766195:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575759.766198:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575759.766200:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575759.766203:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575759.766207:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575759.766216:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766223:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575759.766224:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766227:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575759.766229:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575759.766238:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766243:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575759.766245:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766250:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575759.766251:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766252:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575759.766253:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766255:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575759.766256:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575759.766258:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575759.766260:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575759.766262:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575759.766266:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575759.766273:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393233536:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575759.766278:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766280:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575759.766283:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393233536/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766294:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575759.766297:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575759.766299:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575759.766302:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575759.766304:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575759.766306:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575759.766308:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575759.766310:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:2.0:1788575759.766314:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575759.766316:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575759.766317:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:2.0:1788575759.766319:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575759.766320:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766321:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766327:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575759.766328:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766330:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575759.766331:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575759.766332:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575759.766335:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766339:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575759.766340:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575759.766342:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575759.766343:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575759.766344:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575759.766346:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233664:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575759.766348:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575759.766350:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575759.766351:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575759.766352:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575759.766354:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575759.766355:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575759.766357:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575759.766360:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575759.766362:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575759.766364:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393233664, portal 10
00000100:00000001:2.0:1788575759.766365:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575759.766366:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000040:2.0:1788575759.766370:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766374:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575759.766375:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575759.766376:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575759.766378:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393233664, offset 0
00000400:00000200:2.0:1788575759.766380:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575759.766382:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575759.766385:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575759.766387:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575759.766388:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575759.766390:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575759.766419:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575759.766421:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331081:331081:256:4294967295] 192.168.202.110@tcp LPNI seq info [331038:331038:8:4294967295]
00000400:00000200:2.0:1788575759.766424:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575759.766429:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575759.766432:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575759.766434:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575759.766437:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575759.766439:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788575759.766441:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575759.766444:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:2.0:1788575759.766446:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575759.766447:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575759.766449:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575759.766451:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575759.766453:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575759.766454:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575759.766456:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575759.766459:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766463:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575759.766465:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766469:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766470:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575759.766472:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575759.766474:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575759.766476:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575759.766477:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307363367
00000100:00000001:2.0:1788575759.766479:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307364281
00000100:00000001:2.0:1788575759.766480:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307363367
00000100:00000001:2.0:1788575759.766482:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307364281
00000100:00000001:2.0:1788575759.766483:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575759.766484:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766485:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575759.766487:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575759/real 1788575759]  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575759.766507:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575759.766508:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575759.766509:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575759.766511:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575759.766512:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575759.766513:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575759.766516:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575759.766520:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766533:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575759.766534:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766536:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575759.766537:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575759.766541:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766547:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575759.766551:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766557:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575759.766559:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766561:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575759.766562:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766563:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575759.766566:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575759.766568:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575759.766571:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575759.766574:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575759.766578:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575759.766585:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233664:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575759.766588:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575759.766595:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766600:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575759.766601:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575759.766605:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575759.766606:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575759.766607:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575759.766609:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233792:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575759.766611:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575759.766613:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575759.766614:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575759.766615:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575759.766617:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575759.766619:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575759.766621:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575759.766623:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575759.766625:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575759.766628:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393233792, portal 4
00000100:00000001:2.0:1788575759.766630:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575759.766631:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:2.0:1788575759.766636:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766641:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575759.766644:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575759.766645:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575759.766648:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393233792, offset 0
00000400:00000200:2.0:1788575759.766651:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575759.766655:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575759.766659:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575759.766661:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575759.766664:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575759.766666:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575759.766668:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575759.766671:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331082:331082:256:4294967295] 192.168.202.110@tcp LPNI seq info [331039:331039:8:4294967295]
00000400:00000200:2.0:1788575759.766675:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575759.766680:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575759.766684:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575759.766686:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575759.766688:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575759.766690:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1788575759.766693:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575759.766694:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:2.0:1788575759.766697:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575759.766698:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575759.766700:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575759.766702:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575759.766704:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575759.766705:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575759.766706:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575759.766709:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766713:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575759.766715:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766719:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766720:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575759.766722:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575759.766723:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575759.766725:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575759.766727:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307363513
00000100:00000001:2.0:1788575759.766728:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307364281
00000100:00000001:2.0:1788575759.766730:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307363513
00000100:00000001:2.0:1788575759.766731:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307364281
00000100:00000001:2.0:1788575759.766732:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575759.766734:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766735:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575759.766737:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575759/real 1788575759]  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575759.766742:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575759.766742:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575759.766744:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575759.766745:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575759.766746:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575759.766747:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575759.766748:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575759.766753:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766757:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575759.766757:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766758:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575759.766759:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575759.766761:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766764:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575759.766766:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766771:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575759.766773:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766774:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575759.766775:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766776:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575759.766778:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575759.766780:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575759.766783:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575759.766785:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575759.766788:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575759.766795:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393233792:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575759.766798:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766800:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575759.766803:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393233664/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766809:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575759.766811:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575759.766813:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575759.766816:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575759.766817:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575759.766818:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575759.766820:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575759.766822:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:2.0:1788575759.766824:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575759.766825:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575759.766825:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:2.0:1788575759.766827:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575759.766828:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766829:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575759.766831:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393233792/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575759.766835:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575759.766836:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575759.766837:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575759.766838:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575759.766839:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575759.766840:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575759.766842:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575759.766843:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:2.0:1788575759.766845:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575759.766847:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575759.766848:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:2.0:1788575759.766849:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575759.766850:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766852:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575759.766856:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575759.766857:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575759.766859:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575759.766860:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575760.149487:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575760.149492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575760.149495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575760.149496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575760.149498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575760.149499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575760.149558:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575760.149563:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575760.149566:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575760.277542:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575760.277547:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575760.277549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575760.277553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575760.277555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575760.277558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575760.277560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575760.277562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575760.277564:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575760.725513:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307364281, at 4307364288
00000100:00000040:1.0:1788575760.725513:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307364281, at 4307364288
00000100:00000010:1.0:1788575760.725527:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000100:00000010:0.0:1788575760.725530:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00010000:00000001:3.0:1788575761.173552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575761.173559:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575761.173565:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575761.173566:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575761.173569:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575761.173570:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575761.173625:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575761.173628:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575761.173631:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575761.301558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575761.301565:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575761.301567:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575761.301572:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575761.301574:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575761.301577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575761.301579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575761.301581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575761.301584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575762.197510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575762.197515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575762.197518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575762.197519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575762.197522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575762.197530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575762.197599:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575762.197603:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575762.197606:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575762.325504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575762.325518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575762.325520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575762.325525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575762.325528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575762.325532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575762.325535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575762.325537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575762.325541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575763.221535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575763.221545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575763.221549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575763.221550:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575763.221553:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575763.221554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575763.221628:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575763.221633:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575763.221637:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575763.349519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575763.349527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575763.349530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575763.349535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575763.349538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575763.349542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575763.349545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575763.349547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575763.349550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575764.245528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575764.245537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575764.245542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575764.245543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575764.245546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575764.245546:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575764.245610:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575764.245613:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575764.245616:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575764.373515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575764.373522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575764.373524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575764.373529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575764.373532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575764.373536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575764.373539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575764.373541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575764.373544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575764.885641:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575764.885655:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575764.885659:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575764.885664:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575764.885669:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575764.885677:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12696
00000400:00000200:2.0:1788575764.885687:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575764.885694:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575764.885697:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575764.885701:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575764.885705:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575764.885707:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575764.885710:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575764.885715:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575764.885717:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575764.885721:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575764.885730:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575764.885733:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575764.885735:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575764.885738:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575764.885741:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575764.885743:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575764.885744:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575764.885747:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575764.885751:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575764.885753:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575764.885756:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575764.885759:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575764.885762:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575764.885764:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575764.885767:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575764.885777:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01c00.
00000020:00000040:2.0:1788575764.885781:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575764.885785:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575764.885787:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575764.885790:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575764.885793:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575764.885797:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575764.885807:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000100:00000001:2.0:1788575764.885816:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575764.885826:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575764.885837:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575764.885842:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575764.885846:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000de3acaa1] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575764.885870:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575764.885874:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575764.885878:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575764.885880:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575764.885882:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575764.885885:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575764.885887:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12696
00000400:00000200:2.0:1788575764.885892:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575764.885896:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575764.885899:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575764.885902:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575764.885903:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575764.885905:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575764.885908:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575764.885909:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575764.885911:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575764.885915:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1788575764.885915:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1788575764.885919:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575764.885922:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575764.885925:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000040:3.0:1788575764.885927:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575764.885928:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575764.885934:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575764.885942:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575764.885946:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575764.885947:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575764.885950:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575764.885951:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575764.885954:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575764.885956:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575764.885959:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575764.885964:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575764.885966:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575764.885967:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575764.885971:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575764.885976:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575764.885976:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575764.885981:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575764.885982:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393233920:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575764.885984:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575764.885990:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:2.0:1788575764.885991:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575764.885996:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575764.885997:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1788575764.885998:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575764.886002:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575764.886004:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00e00.
02000000:00000001:3.0:1788575764.886007:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1788575764.886008:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575764.886012:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575764.886014:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575764.886015:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:2.0:1788575764.886016:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575764.886020:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575764.886020:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575764.886023:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575764.886030:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000400:00000200:3.0:1788575764.886031:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575764.886036:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575764.886038:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00100000:2.0:1788575764.886045:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575764.886049:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575764.886057:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575764.886060:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575764.886060:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393233920, portal 25
00000100:00000001:3.0:1788575764.886067:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000040:2.0:1788575764.886068:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5f665f0] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886070:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000001:2.0:1788575764.886079:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575764.886083:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575764.886088:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575764.886089:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575764.886092:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575764.886095:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575764.886101:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575764.886106:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12696
00000100:00000001:3.0:1788575764.886106:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575764.886112:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575764.886115:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575764.886118:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000400:00000200:2.0:1788575764.886119:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575764.886124:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000200:3.0:1788575764.886126:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393233920, offset 0
00000100:00000040:2.0:1788575764.886127:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575764.886132:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575764.886134:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575764.886139:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575764.886140:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:2.0:1788575764.886144:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575764.886147:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575764.886149:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575764.886151:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575764.886154:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1788575764.886156:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575764.886160:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575764.886164:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575764.886167:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1788575764.886168:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575764.886181:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575764.886183:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788575764.886187:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575764.886191:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:2.0:1788575764.886192:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575764.886198:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1788575764.886199:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575764.886204:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575764.886206:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:2.0:1788575764.886208:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575764.886213:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575764.886213:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331083:331083:256:4294967295] 192.168.202.110@tcp LPNI seq info [331040:331040:8:4294967295]
02000000:00000001:2.0:1788575764.886217:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575764.886221:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575764.886223:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575764.886228:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575764.886228:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575764.886232:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575764.886237:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03100.
00000020:00000040:2.0:1788575764.886252:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575764.886257:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1788575764.886258:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575764.886262:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575764.886265:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575764.886269:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.0:1788575764.886269:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788575764.886273:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1788575764.886278:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
02000000:00000010:2.0:1788575764.886281:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12c00.
00000800:00000200:3.0:1788575764.886288:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575764.886294:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575764.886295:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00100000:2.0:1788575764.886303:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1788575764.886305:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575764.886311:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:3.0:1788575764.886320:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575764.886325:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000100:00000001:2.0:1788575764.886329:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575764.886332:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575764.886334:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1788575764.886339:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008692924c] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575764.886345:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575764.886355:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575764.886359:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:2.0:1788575764.886361:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575764.886364:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:2.0:1788575764.886369:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1788575764.886371:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575764.886377:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575764.886387:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12706)
00000100:00000001:3.0:1788575764.886389:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575764.886415:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575789 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886423:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575764.886425:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575764.886432:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575764.886437:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575764.886440:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307369402
00000100:00000010:3.0:1788575764.886447:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575764.886455:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575764.886468:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307369402
00000100:00000010:3.0:1788575764.886473:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575764.886477:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575764.886483:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575764.886487:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575764.886490:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575764.886496:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575764/real 1788575764]  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575764.886507:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575764.886509:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575764.886512:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575764.886515:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575764.886517:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575764.886519:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575764.886523:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575764.886531:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886538:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575764.886540:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.886543:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575764.886545:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575764.886549:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886557:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575764.886563:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886570:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575764.886573:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.886575:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575764.886576:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.886579:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575764.886581:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575764.886584:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575764.886589:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575764.886591:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575764.886595:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575764.886604:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393233920:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575764.886610:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.886612:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575764.886615:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393233920/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886622:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575764.886624:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575764.886627:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfbc00.
02000000:00000001:3.0:1788575764.886631:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575764.886633:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575764.886635:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575764.886637:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575764.886649:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575764.886653:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575764.886655:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575764.886657:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:3.0:1788575764.886661:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575764.886662:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.886665:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.886673:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575764.886675:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575764.886677:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575764.886678:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575764.886680:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575764.886684:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886691:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575764.886692:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575764.886695:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575764.886697:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575764.886699:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575764.886701:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575764.886705:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575764.886707:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575764.886708:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575764.886710:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575764.886713:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfbc00.
02000000:00000001:3.0:1788575764.886715:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575764.886718:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575764.886721:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575764.886734:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575764.886737:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393234048, portal 10
00000100:00000001:3.0:1788575764.886739:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575764.886740:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:3.0:1788575764.886747:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886753:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575764.886756:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575764.886758:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575764.886760:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393234048, offset 0
00000400:00000200:3.0:1788575764.886765:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575764.886769:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575764.886772:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575764.886775:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575764.886778:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575764.886780:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575764.886782:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575764.886785:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331084:331084:256:4294967295] 192.168.202.110@tcp LPNI seq info [331041:331041:8:4294967295]
00000400:00000200:3.0:1788575764.886790:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575764.886797:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575764.886802:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575764.886805:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575764.886809:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575764.886812:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1788575764.886817:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575764.886819:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:3.0:1788575764.886823:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575764.886825:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575764.886828:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575764.886832:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575764.886835:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575764.886837:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575764.886839:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575764.886842:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886850:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575764.886853:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575789 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886859:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575764.886861:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575764.886865:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575764.886869:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575764.886872:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575764.886874:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307368487
00000100:00000001:3.0:1788575764.886877:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307369402
00000100:00000001:3.0:1788575764.886880:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307368487
00000100:00000001:3.0:1788575764.886882:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307369402
00000100:00000001:3.0:1788575764.886884:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575764.886886:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575764.886888:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575764.886892:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575764/real 1788575764]  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575764.886900:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575764.886901:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575764.886903:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575764.886905:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575764.886907:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575764.886909:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575764.886913:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575764.886921:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886927:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575764.886928:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.886930:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575764.886932:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575764.886935:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886941:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575764.886945:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886951:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575764.886954:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.886956:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575764.886957:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.886958:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575764.886961:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575764.886963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575764.886966:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575764.886968:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575764.886972:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575764.886979:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575764.886983:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575764.886986:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.886992:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575764.886994:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575764.886996:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575764.886997:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575764.886999:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575764.887002:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575764.887005:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575764.887007:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575764.887008:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575764.887010:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575764.887013:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575764.887015:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575764.887018:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575764.887021:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575764.887024:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575764.887027:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393234176, portal 4
00000100:00000001:3.0:1788575764.887029:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575764.887030:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:3.0:1788575764.887037:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.887043:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575764.887045:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575764.887047:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575764.887050:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393234176, offset 0
00000400:00000200:3.0:1788575764.887055:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575764.887058:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575764.887062:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575764.887065:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575764.887067:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575764.887069:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575764.887072:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575764.887075:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331085:331085:256:4294967295] 192.168.202.110@tcp LPNI seq info [331042:331042:8:4294967295]
00000400:00000200:3.0:1788575764.887079:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575764.887085:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575764.887089:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575764.887092:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575764.887096:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575764.887098:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1788575764.887102:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575764.887106:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:3.0:1788575764.887110:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575764.887112:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575764.887115:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575764.887118:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575764.887121:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575764.887123:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575764.887125:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575764.887128:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575789 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.887136:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575764.887139:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575789 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.887145:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575764.887147:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575764.887150:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575764.887153:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575764.887156:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575764.887158:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307368633
00000100:00000001:3.0:1788575764.887161:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307369402
00000100:00000001:3.0:1788575764.887163:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307368633
00000100:00000001:3.0:1788575764.887165:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307369402
00000100:00000001:3.0:1788575764.887167:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575764.887169:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575764.887171:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575764.887175:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575764/real 1788575764]  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575764.887182:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575764.887184:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575764.887185:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575764.887188:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575764.887189:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575764.887191:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575764.887195:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575764.887203:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.887209:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575764.887210:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.887212:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575764.887213:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575764.887216:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.887223:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575764.887226:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.887233:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575764.887236:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.887252:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575764.887253:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.887255:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575764.887257:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575764.887260:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575764.887263:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575764.887266:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575764.887269:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575764.887277:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575764.887281:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.887283:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575764.887286:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393234048/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.887292:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575764.887294:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575764.887296:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfbc00.
02000000:00000001:3.0:1788575764.887298:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575764.887300:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575764.887301:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575764.887304:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575764.887305:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575764.887308:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575764.887309:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575764.887311:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:3.0:1788575764.887313:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575764.887314:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.887316:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575764.887319:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393234176/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575789 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575764.887333:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575764.887334:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575764.887335:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8c00.
02000000:00000001:3.0:1788575764.887337:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575764.887339:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575764.887340:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575764.887342:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575764.887343:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575764.887346:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575764.887347:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575764.887348:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:3.0:1788575764.887351:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575764.887352:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.887354:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575764.887358:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575764.887359:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575764.887361:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575764.887362:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575765.269472:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575765.269480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575765.269485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575765.269488:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575765.269490:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575765.269493:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575765.269496:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575765.269498:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575765.269500:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575765.269502:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575765.269505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575765.269506:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575765.269507:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575765.269509:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575765.269510:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575765.269511:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575765.269512:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575765.269514:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575765.269552:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575765.269556:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575765.269558:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575765.397541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575765.397551:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575765.397553:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575765.397560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575765.397564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575765.397569:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575765.397572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575765.397574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575765.397576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575765.845536:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307369402, at 4307369408
00000100:00000040:1.0:1788575765.845536:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307369402, at 4307369408
00000100:00000010:1.0:1788575765.845553:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575765.845556:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00010000:00000001:3.0:1788575766.293504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575766.293510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575766.293515:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575766.293516:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575766.293519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575766.293520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575766.293578:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575766.293581:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575766.293583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575766.421462:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575766.421474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575766.421478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575766.421486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575766.421490:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575766.421496:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575766.421499:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575766.421502:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575766.421506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575767.317502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575767.317507:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575767.317511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575767.317512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575767.317514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575767.317515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575767.317577:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575767.317580:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575767.317583:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575767.445548:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575767.445556:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575767.445559:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575767.445565:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575767.445569:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575767.445573:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575767.445576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575767.445579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575767.445582:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575768.341473:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575768.341480:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575768.341482:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575768.341484:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575768.341487:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575768.341488:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575768.341490:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575768.341491:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575768.341493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575768.341494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575768.341497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575768.341498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575768.341544:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575768.341547:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575768.341549:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575768.469456:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575768.469463:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575768.469465:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575768.469471:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575768.469475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575768.469478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575768.469481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575768.469483:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575768.469486:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575769.365509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575769.365515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575769.365521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575769.365522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575769.365525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575769.365527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575769.365633:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575769.365641:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575769.365647:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575769.493508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575769.493513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575769.493515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575769.493519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575769.493522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575769.493525:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575769.493527:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575769.493529:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575769.493531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575770.005490:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575770.005501:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575770.005503:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575770.005508:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575770.005513:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575770.005520:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12701
00000400:00000200:3.0:1788575770.005529:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575770.005535:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575770.005538:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575770.005540:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005543:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575770.005544:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005546:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575770.005549:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575770.005550:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575770.005552:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575770.005554:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575770.005555:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575770.005557:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575770.005559:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005560:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575770.005561:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005562:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575770.005564:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575770.005567:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005568:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575770.005570:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575770.005572:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575770.005574:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575770.005576:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575770.005578:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575770.005586:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0a80.
00000020:00000040:3.0:1788575770.005588:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575770.005591:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575770.005592:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575770.005594:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575770.005597:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575770.005601:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575770.005609:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8c00.
00000100:00000001:3.0:1788575770.005617:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575770.005626:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575770.005637:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575770.005640:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575770.005644:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b298c26f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575770.005660:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575770.005662:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575770.005665:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575770.005666:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575770.005667:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575770.005669:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575770.005671:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12701
00000400:00000200:3.0:1788575770.005674:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575770.005677:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575770.005678:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575770.005680:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005682:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575770.005683:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575770.005685:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575770.005685:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575770.005689:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575770.005690:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575770.005691:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575770.005692:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575770.005695:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575770.005697:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575770.005697:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575770.005698:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575770.005702:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005704:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575770.005708:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575770.005709:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575770.005709:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575770.005713:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575770.005717:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005720:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575770.005723:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575770.005729:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1788575770.005729:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575770.005733:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575770.005733:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575770.005738:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575770.005740:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:3.0:1788575770.005742:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575770.005744:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1788575770.005745:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575770.005749:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575770.005751:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575770.005752:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2300.
00000020:00000040:3.0:1788575770.005755:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00100000:2.0:1788575770.005758:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393234304:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575770.005758:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575770.005760:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575770.005763:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575770.005765:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575770.005769:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575770.005769:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575770.005772:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575770.005774:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1788575770.005777:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575770.005778:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575770.005782:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:3.0:1788575770.005783:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575770.005789:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000010:2.0:1788575770.005790:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
00000100:00000001:3.0:1788575770.005790:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575770.005796:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575770.005796:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc4bce75] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575770.005805:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575770.005808:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575770.005809:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575770.005814:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575770.005814:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:2.0:1788575770.005816:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00080000:3.0:1788575770.005817:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575770.005819:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575770.005823:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12701
00000400:00000010:2.0:1788575770.005825:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000400:00000200:3.0:1788575770.005826:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575770.005830:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575770.005832:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575770.005834:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005835:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:2.0:1788575770.005836:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393234304, portal 25
00000100:00000040:3.0:1788575770.005836:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575770.005842:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575770.005842:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575770.005845:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000020:00000001:3.0:1788575770.005846:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575770.005848:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575770.005850:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575770.005852:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000040:2.0:1788575770.005856:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1788575770.005856:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575770.005860:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575770.005868:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005870:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575770.005871:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575770.005873:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575770.005876:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:2.0:1788575770.005877:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575770.005881:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:2.0:1788575770.005882:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00080000:3.0:1788575770.005886:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:2.0:1788575770.005887:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393234304, offset 0
00000100:00000001:3.0:1788575770.005888:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575770.005891:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575770.005893:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575770.005895:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575770.005896:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575770.005899:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575770.005900:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575770.005902:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575770.005903:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575770.005906:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1788575770.005909:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000020:00000040:3.0:1788575770.005911:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575770.005913:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575770.005916:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575770.005919:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575770.005920:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575770.005921:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:3.0:1788575770.005924:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575770.005928:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000010:3.0:1788575770.005930:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000400:00000200:2.0:1788575770.005932:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575770.005937:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575770.005937:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575770.005942:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575770.005947:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331086:331086:256:4294967295] 192.168.202.110@tcp LPNI seq info [331043:331043:8:4294967295]
00000100:00100000:3.0:1788575770.005948:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575770.005953:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575770.005962:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575770.005964:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1788575770.005965:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575770.005973:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788575770.005974:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575770.005982:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000100:00000001:3.0:1788575770.005982:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575770.005986:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1788575770.006001:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575770.006002:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:2.0:1788575770.006008:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000040:3.0:1788575770.006010:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12711)
00000800:00000200:2.1:1788575770.006014:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575770.006018:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1788575770.006020:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575770.006022:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575770.006028:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575770.006031:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575770.006036:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575770.006038:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575770.006041:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575770.006044:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006050:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575770.006052:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006057:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575770.006058:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575770.006063:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575770.006066:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575770.006068:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307374521
00000100:00000010:2.0:1788575770.006077:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c0c0.
00000100:00000001:2.0:1788575770.006082:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575770.006086:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307374521
00000100:00000010:2.0:1788575770.006088:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ca20.
00000100:00000001:2.0:1788575770.006090:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575770.006093:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575770.006094:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575770.006096:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575770.006099:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575770/real 1788575770]  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575770.006105:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575770.006106:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575770.006108:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575770.006110:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575770.006111:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575770.006112:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575770.006114:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575770.006119:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006123:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575770.006124:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006126:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575770.006127:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575770.006129:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006133:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575770.006136:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006140:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575770.006142:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006143:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575770.006144:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006146:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575770.006147:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575770.006149:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575770.006152:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575770.006153:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575770.006155:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575770.006160:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393234304:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575770.006165:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006166:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575770.006168:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393234304/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006172:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575770.006173:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575770.006175:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575770.006176:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575770.006178:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575770.006179:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575770.006180:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575770.006182:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:2.0:1788575770.006184:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575770.006185:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575770.006187:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:2.0:1788575770.006188:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575770.006189:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006191:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006196:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575770.006197:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575770.006199:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575770.006200:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575770.006201:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575770.006203:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006207:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575770.006208:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575770.006209:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575770.006211:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575770.006212:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575770.006213:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234432:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575770.006215:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575770.006216:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575770.006217:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575770.006218:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575770.006220:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575770.006221:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575770.006224:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575770.006225:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575770.006227:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575770.006229:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393234432, portal 10
00000100:00000001:2.0:1788575770.006231:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575770.006231:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000040:2.0:1788575770.006236:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006240:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575770.006241:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575770.006242:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575770.006244:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393234432, offset 0
00000400:00000200:2.0:1788575770.006246:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575770.006248:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575770.006250:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575770.006252:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575770.006254:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575770.006255:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575770.006257:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575770.006259:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331087:331087:256:4294967295] 192.168.202.110@tcp LPNI seq info [331044:331044:8:4294967295]
00000400:00000200:2.0:1788575770.006261:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575770.006265:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575770.006268:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575770.006269:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575770.006272:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575770.006273:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575770.006276:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575770.006277:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1788575770.006279:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575770.006280:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575770.006282:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575770.006284:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575770.006286:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575770.006287:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575770.006288:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575770.006290:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006294:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575770.006296:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006300:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575770.006301:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575770.006302:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575770.006305:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575770.006307:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575770.006308:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307373606
00000100:00000001:2.0:1788575770.006309:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307374521
00000100:00000001:2.0:1788575770.006311:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307373606
00000100:00000001:2.0:1788575770.006312:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307374521
00000100:00000001:2.0:1788575770.006314:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575770.006315:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575770.006316:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575770.006318:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575770/real 1788575770]  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575770.006322:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575770.006322:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575770.006323:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575770.006324:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575770.006325:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575770.006326:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575770.006328:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575770.006333:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006336:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575770.006337:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006338:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575770.006339:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575770.006340:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006344:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575770.006346:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006349:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575770.006351:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006351:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575770.006352:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006353:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575770.006354:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575770.006356:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575770.006357:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575770.006359:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575770.006361:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575770.006365:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234432:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575770.006367:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575770.006369:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006377:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575770.006378:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575770.006379:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575770.006380:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575770.006381:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575770.006383:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234560:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575770.006384:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575770.006385:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575770.006386:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575770.006387:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575770.006388:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575770.006389:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575770.006407:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575770.006409:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575770.006411:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575770.006413:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393234560, portal 4
00000100:00000001:2.0:1788575770.006414:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575770.006415:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575770.006419:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006423:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575770.006424:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575770.006425:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575770.006427:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393234560, offset 0
00000400:00000200:2.0:1788575770.006429:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575770.006431:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575770.006433:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575770.006434:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575770.006436:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575770.006437:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575770.006438:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575770.006441:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331088:331088:256:4294967295] 192.168.202.110@tcp LPNI seq info [331045:331045:8:4294967295]
00000400:00000200:2.0:1788575770.006443:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575770.006447:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575770.006449:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575770.006451:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575770.006453:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575770.006455:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1788575770.006457:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575770.006459:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1788575770.006461:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575770.006462:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:2.0:1788575770.006464:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575770.006466:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575770.006467:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:2.0:1788575770.006468:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:2.0:1788575770.006469:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575770.006471:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006476:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575770.006477:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006481:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575770.006482:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575770.006484:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575770.006486:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575770.006489:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575770.006490:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307373753
00000100:00000001:2.0:1788575770.006493:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307374521
00000100:00000001:2.0:1788575770.006494:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307373753
00000100:00000001:2.0:1788575770.006496:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307374521
00000100:00000001:2.0:1788575770.006498:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575770.006500:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575770.006502:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575770.006505:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575770/real 1788575770]  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575770.006513:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575770.006514:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575770.006516:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575770.006518:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575770.006520:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575770.006522:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575770.006525:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575770.006534:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006539:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575770.006540:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006541:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575770.006542:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575770.006544:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006548:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575770.006550:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006554:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575770.006555:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006556:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575770.006557:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006558:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575770.006559:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575770.006560:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575770.006563:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575770.006564:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575770.006566:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575770.006570:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234560:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575770.006572:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006573:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575770.006575:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393234432/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006578:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575770.006579:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575770.006580:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575770.006582:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575770.006583:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575770.006584:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575770.006585:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575770.006586:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575770.006588:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575770.006588:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575770.006589:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:2.0:1788575770.006591:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575770.006592:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006594:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575770.006597:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393234560/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575770.006603:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575770.006604:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575770.006605:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575770.006607:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575770.006608:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575770.006609:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575770.006611:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575770.006613:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:2.0:1788575770.006616:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575770.006617:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575770.006618:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575770.006620:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575770.006621:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006623:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.006626:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575770.006627:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575770.006629:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575770.006631:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575770.389482:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575770.389488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575770.389494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575770.389495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575770.389499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575770.389500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575770.389551:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575770.389555:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575770.389557:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575770.517523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575770.517529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575770.517531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575770.517536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575770.517539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575770.517543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.517545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575770.517548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575770.517551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575770.965473:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307374521, at 4307374528
00000100:00000040:1.0:1788575770.965473:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307374521, at 4307374528
00000100:00000010:1.0:1788575770.965487:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000100:00000010:0.0:1788575770.965489:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00010000:00000001:3.0:1788575771.413508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575771.413515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575771.413520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575771.413521:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575771.413524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575771.413526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575771.413580:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575771.413584:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575771.413587:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575771.541470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575771.541475:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575771.541476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575771.541481:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575771.541484:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575771.541487:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575771.541488:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575771.541491:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575771.541492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575772.437514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575772.437520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575772.437525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575772.437527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575772.437530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575772.437531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575772.437587:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575772.437590:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575772.437592:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575772.565524:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575772.565531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575772.565533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575772.565539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575772.565542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575772.565547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575772.565550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575772.565553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575772.565556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575773.461513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575773.461519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575773.461524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575773.461525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575773.461528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575773.461530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575773.461583:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575773.461586:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575773.461588:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575773.589527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575773.589534:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575773.589537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575773.589542:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575773.589546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575773.589550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575773.589553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575773.589556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575773.589558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575774.485474:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575774.485480:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575774.485485:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575774.485487:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575774.485491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575774.485492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575774.485551:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575774.485555:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575774.485558:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575774.613487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575774.613494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575774.613496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575774.613501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575774.613503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575774.613506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575774.613508:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575774.613510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575774.613512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575775.125587:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575775.125598:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575775.125601:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575775.125605:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575775.125610:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575775.125616:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12706
00000400:00000200:0.0:1788575775.125624:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575775.125630:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575775.125633:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575775.125636:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125639:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575775.125641:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125643:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575775.125647:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575775.125649:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575775.125652:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575775.125655:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575775.125657:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575775.125660:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575775.125662:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125664:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575775.125666:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125668:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575775.125671:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575775.125675:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125678:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575775.125681:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575775.125684:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575775.125686:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575775.125689:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575775.125691:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575775.125697:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788575775.125701:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575775.125713:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575775.125714:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575775.125717:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575775.125719:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575775.125723:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575775.125730:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575775.125737:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575775.125760:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575775.125770:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575775.125774:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575775.125778:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575775.125805:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575775.125808:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575775.125811:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575775.125813:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575775.125814:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575775.125817:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575775.125820:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12706
00000400:00000200:0.0:1788575775.125824:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575775.125827:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575775.125829:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575775.125832:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125834:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575775.125836:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125838:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575775.125840:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575775.125841:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575775.125843:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575775.125845:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575775.125846:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575775.125849:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575775.125851:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125853:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575775.125855:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125858:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575775.125860:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575775.125863:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125865:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575775.125867:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575775.125872:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575775.125874:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575775.125875:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575775.125879:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575775.125882:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575775.125883:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575775.125887:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788575775.125889:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575775.125892:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575775.125893:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575775.125894:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575775.125896:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575775.125897:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575775.125901:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575775.125905:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575775.125909:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575775.125917:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575775.125919:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575775.125922:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575775.125930:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575775.125933:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575775.125935:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575775.125937:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575775.125938:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575775.125941:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575775.125944:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12706
00000400:00000200:0.0:1788575775.125947:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575775.125950:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575775.125953:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575775.125955:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125957:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575775.125959:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125961:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575775.125963:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575775.125964:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575775.125966:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575775.125968:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575775.125970:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575775.125972:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575775.125974:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125976:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575775.125978:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125980:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575775.125982:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575775.125985:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575775.125987:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575775.125989:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575775.125991:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575775.125993:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575775.125995:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575775.125997:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575775.125999:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575775.126000:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575775.126004:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788575775.126007:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575775.126009:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575775.126010:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575775.126012:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575775.126013:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575775.126014:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575775.126018:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788575775.126021:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575775.126026:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575775.126033:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575775.126035:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575775.126038:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575775.126046:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575775.126048:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575775.126052:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575775.126066:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12716)
00000100:00000001:2.0:1788575775.126822:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575775.126827:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575775.126830:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575775.126838:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575775.126848:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575775.126850:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575775.126854:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575775.126857:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575775.126860:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575775.126865:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393234688:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575775.126869:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575775.126873:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575775.126876:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575775.126878:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575775.126883:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575775.126885:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575775.126891:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575775.126894:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575775.126901:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575775.126907:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393234688, portal 25
00000100:00000001:2.0:1788575775.126909:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575775.126911:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523505920 : -130484186045696 : ffff895348712300)
00000100:00000040:2.0:1788575775.126917:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575775.126925:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575775.126928:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575775.126931:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788575775.126935:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393234688, offset 0
00000400:00000200:2.0:1788575775.126941:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575775.126948:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575775.126954:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575775.126960:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575775.126964:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575775.126967:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575775.126970:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575775.126973:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331089:331089:256:4294967295] 192.168.202.110@tcp LPNI seq info [331046:331046:8:4294967295]
00000400:00000200:2.0:1788575775.126977:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575775.126986:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575775.126991:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575775.126996:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575775.127002:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575775.127007:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575775.127025:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575775.127029:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575775.127032:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307379642
00000100:00000010:2.0:1788575775.127036:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c480.
00000100:00000001:2.0:1788575775.127042:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575775.127053:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307379642
00000100:00000010:2.0:1788575775.127057:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd80.
00000100:00000001:2.0:1788575775.127060:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575775.127063:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575775.127066:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575775.127069:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575775.127073:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575775.127081:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575775.127082:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575775.127085:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575775.127087:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575775.127090:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575775.127094:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234816:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575775.127097:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575775.127099:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575775.127101:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575775.127102:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575775.127105:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575775.127108:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575775.127111:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575775.127114:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575775.127118:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575775.127121:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393234816, portal 10
00000100:00000001:2.0:1788575775.127123:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575775.127125:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523500544 : -130484186051072 : ffff895348710e00)
00000100:00000040:2.0:1788575775.127131:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575775.127137:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575775.127140:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575775.127144:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c654c8.
00000100:00000200:2.0:1788575775.127152:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393234816, offset 0
00000400:00000200:2.0:1788575775.127158:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575775.127163:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575775.127166:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575775.127169:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575775.127171:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575775.127173:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575775.127176:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575775.127179:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331090:331090:255:4294967295] 192.168.202.110@tcp LPNI seq info [331047:331047:7:4294967295]
00000400:00000200:2.0:1788575775.127184:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575775.127191:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575775.127196:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575775.127201:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133800.
00000800:00000200:2.0:1788575775.127205:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575775.127210:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575775.127212:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575775.127215:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575775.127218:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575775.127220:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307378727
00000100:00000001:2.0:1788575775.127223:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307379642
00000100:00000001:2.0:1788575775.127225:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307378727
00000100:00000001:2.0:1788575775.127227:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307379642
00000100:00000001:2.0:1788575775.127229:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575775.127230:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575775.127233:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575775.127236:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575775.127243:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575775.127244:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575775.127246:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575775.127248:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575775.127250:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575775.127252:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575775.127256:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575775.127257:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575775.127259:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575775.127260:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575775.127263:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575775.127265:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575775.127268:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575775.127271:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575775.127274:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575775.127277:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393234944, portal 4
00000100:00000001:2.0:1788575775.127279:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575775.127280:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:2.0:1788575775.127286:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575775.127293:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575775.127295:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575775.127297:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65110.
00000100:00000200:2.0:1788575775.127300:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393234944, offset 0
00000400:00000200:2.0:1788575775.127304:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575775.127308:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575775.127311:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575775.127313:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575775.127315:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575775.127318:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575775.127320:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575775.127323:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331091:331091:254:4294967295] 192.168.202.110@tcp LPNI seq info [331048:331048:6:4294967295]
00000400:00000200:2.0:1788575775.127326:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575775.127332:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575775.127336:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575775.127340:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575775.127343:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575775.127347:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575775.127350:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575775.127352:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575775.127355:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575775.127357:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307378873
00000100:00000001:2.0:1788575775.127359:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307379642
00000100:00000001:2.0:1788575775.127362:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307378873
00000100:00000001:2.0:1788575775.127363:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307379642
00000100:00000001:2.0:1788575775.127365:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575775.127367:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575775.127370:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575775.127373:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575775.127381:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575775.127384:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575775.127386:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575775.127387:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575775.127388:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575775.127390:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575775.509513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575775.509520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575775.509525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575775.509528:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575775.509530:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575775.509534:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575775.509537:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575775.509539:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575775.509541:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575775.509543:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575775.509547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575775.509548:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575775.509550:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575775.509551:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575775.509553:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575775.509554:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575775.509556:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575775.509558:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575775.509757:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575775.509761:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575775.509764:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575775.637533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575775.637542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575775.637544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575775.637549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575775.637552:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575775.637556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575775.637559:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575775.637561:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575775.637563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575776.085534:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307379642, at 4307379648
00000100:00000010:1.0:1788575776.085555:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00000100:00000040:0.0:1788575776.085716:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307379642, at 4307379648
00000100:00000010:0.0:1788575776.085735:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00010000:00000001:3.0:1788575776.533542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575776.533555:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575776.533562:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575776.533564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575776.533569:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575776.533570:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575776.533643:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575776.533647:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575776.533651:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575776.661543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575776.661553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575776.661555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575776.661562:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575776.661567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575776.661572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575776.661575:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575776.661578:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575776.661581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575777.557551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575777.557559:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575777.557564:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575777.557566:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575777.557570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575777.557571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575777.557650:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575777.557655:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575777.557659:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575777.685500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575777.685509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575777.685511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575777.685516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575777.685520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575777.685524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575777.685526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575777.685529:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575777.685531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1788575778.197772:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:0.0:1788575778.197831:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1788575778.197842:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575778.197847:0:2350:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:0.0:1788575778.197852:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575778.197857:0:2350:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1788575778.197863:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788575778.197868:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:0.0:1788575778.197874:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788575778.197879:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788575778.197899:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65f68
00000400:00000010:0.0:1788575778.197903:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65f68.
00000100:00000001:0.0:1788575778.197910:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575778.197922:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575778.197958:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575778.197963:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575778.197971:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575778.197974:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788575778.197986:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788575778.197988:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133800.
00000400:00000200:0.0:1788575778.197991:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788575778.197994:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788575778.197997:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c654c8
00000400:00000010:0.0:1788575778.197999:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c654c8.
00000100:00000001:0.0:1788575778.198002:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575778.198006:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575778.198015:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575778.198018:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575778.198025:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575778.198026:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:0.0:1788575778.198031:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1788575778.198034:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:0.0:1788575778.198037:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1788575778.198040:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1788575778.198042:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65110
00000400:00000010:0.0:1788575778.198044:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65110.
00000100:00000001:0.0:1788575778.198047:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575778.198051:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575778.198058:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575778.198061:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575800 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575778.198068:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575778.198069:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575778.198612:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575778.198623:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575778.198628:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575778.198641:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575775/real 1788575778]  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575778.198660:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575778.198663:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575778.198668:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575778.198675:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575778.198677:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575778.198682:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575778.198686:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575778.198695:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.198702:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575778.198705:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.198710:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575778.198713:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575778.198718:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.198724:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575778.198731:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.198744:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575778.198748:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.198751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575778.198752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.198756:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575778.198758:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575778.198763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575778.198768:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575778.198771:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575778.198775:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575778.198785:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393234688:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575778.198791:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575778.198795:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575775/real 1788575778]  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575778.198804:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575778.198805:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575778.198808:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575778.198811:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575778.198813:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575778.198815:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575778.198819:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575778.198827:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.198844:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575778.198845:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.198848:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575778.198849:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575778.198853:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.198860:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575778.198863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.198869:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575778.198871:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.198873:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575778.198874:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.198876:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575778.198879:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575778.198882:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575778.198887:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575778.198890:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575778.198893:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575778.198901:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234816:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575778.198905:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575778.198909:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575775/real 1788575778]  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575778.198925:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575778.198926:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575778.198928:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575778.198931:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575778.198932:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575778.198935:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575778.198938:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575778.198945:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.198951:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575778.198952:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.198954:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575778.198955:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575778.198958:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.198965:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575778.198968:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.198974:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575778.198976:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.198978:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575778.198979:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.198981:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575778.198983:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575778.198986:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575778.198991:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575778.198993:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575778.198997:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575778.199004:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393234944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575778.199009:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.199013:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575778.199016:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712300 x1875444393234688/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.199024:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575778.199029:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575778.199035:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575778.199041:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575778.199046:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575778.199049:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575778.199052:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575778.199056:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:2.0:1788575778.199062:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575778.199065:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575778.199068:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712300.
00000100:00000001:2.0:1788575778.199073:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575778.199075:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.199076:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575778.199080:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710e00 x1875444393234816/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.199086:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575778.199087:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575778.199089:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575778.199091:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575778.199093:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575778.199094:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575778.199096:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575778.199098:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:2.0:1788575778.199101:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575778.199102:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575778.199104:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710e00.
00000100:00000001:2.0:1788575778.199108:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575778.199109:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.199111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575778.199114:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393234944/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575800 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575778.199121:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575778.199122:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575778.199126:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575778.199128:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575778.199129:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575778.199131:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575778.199133:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575778.199134:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:2.0:1788575778.199137:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575778.199138:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575778.199139:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:2.0:1788575778.199142:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575778.199143:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.199146:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575778.199159:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575778.199161:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575778.199164:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575778.199166:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575778.581517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575778.581523:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575778.581525:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575778.581527:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575778.581531:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575778.581532:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575778.581533:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575778.581535:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575778.581537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575778.581538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575778.581540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575778.581541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575778.581602:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575778.581604:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575778.581606:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575778.709520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575778.709526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575778.709527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575778.709533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575778.709537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575778.709541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575778.709543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575778.709546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575778.709548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575779.605445:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575779.605450:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575779.605455:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575779.605455:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575779.605458:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575779.605459:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575779.605507:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575779.605509:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575779.605512:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575779.733490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575779.733495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575779.733497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575779.733502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575779.733505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575779.733508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575779.733510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575779.733512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575779.733525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575780.245544:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575780.245557:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575780.245560:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575780.245565:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575780.245571:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575780.245581:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12711
00000400:00000200:0.0:1788575780.245593:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575780.245600:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575780.245603:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575780.245607:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.245610:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575780.245613:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.245615:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575780.245619:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575780.245622:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575780.245625:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575780.245628:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575780.245631:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575780.245633:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575780.245636:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.245638:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575780.245640:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.245642:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575780.245645:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575780.245650:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575780.245652:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575780.245656:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575780.245660:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575780.245663:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575780.245665:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575780.245668:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575780.245678:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575780.245681:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575780.245686:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575780.245688:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575780.245691:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575780.245695:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575780.245700:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575780.245708:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788575780.245716:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575780.245727:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575780.245738:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575780.245743:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575780.245747:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575780.245779:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575780.245788:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575780.245792:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575780.245794:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575780.245796:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575780.245799:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575780.245801:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12711
00000400:00000200:0.0:1788575780.245805:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575780.245808:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575780.245811:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575780.245815:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.245817:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575780.245820:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.245822:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575780.245824:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575780.245826:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575780.245828:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575780.245830:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575780.245832:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575780.245834:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788575780.245835:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575780.245840:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575780.245842:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575780.245843:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575780.245847:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575780.245848:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.245853:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575780.245855:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:2.0:1788575780.245860:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788575780.245861:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575780.245865:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575780.245869:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575780.245876:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788575780.245878:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575780.245878:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575780.245882:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575780.245885:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575780.245887:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:0.0:1788575780.245891:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575780.245891:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:0.0:1788575780.245897:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575780.245898:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575780.245900:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575780.245905:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393235072:192.168.202.110@tcp:250:kworker.0
00000100:00000010:0.0:1788575780.245906:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788575780.245910:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575780.245913:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575780.245915:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575780.245918:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575780.245920:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575780.245921:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575780.245922:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575780.245924:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575780.245927:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575780.245930:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788575780.245932:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788575780.245937:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575780.245941:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
00000100:00100000:0.0:1788575780.245945:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575780.245945:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575780.245956:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575780.245958:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575780.245960:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575780.245965:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00000040:0.0:1788575780.245966:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575780.245976:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000001:0.0:1788575780.245977:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575780.245982:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575780.245987:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575780.245989:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:2.0:1788575780.245990:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393235072, portal 25
00000100:00080000:0.0:1788575780.245991:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575780.245994:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575780.245996:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:2.0:1788575780.245999:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00080000:0.0:1788575780.246001:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12711
00000400:00000200:0.0:1788575780.246007:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575780.246013:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:2.0:1788575780.246013:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788575780.246017:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575780.246022:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.246027:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575780.246031:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575780.246034:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788575780.246035:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575780.246039:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575780.246041:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000040:2.0:1788575780.246041:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788575780.246045:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:2.0:1788575780.246046:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000020:00000040:0.0:1788575780.246047:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575780.246052:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575780.246054:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:2.0:1788575780.246054:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393235072, offset 0
00000100:00000040:0.0:1788575780.246060:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.246062:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575780.246067:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788575780.246068:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575780.246072:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575780.246074:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575780.246077:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575780.246081:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575780.246082:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575780.246085:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575780.246089:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575780.246091:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575780.246094:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788575780.246095:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575780.246099:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575780.246103:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575780.246104:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575780.246107:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575780.246112:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000010:0.0:1788575780.246113:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788575780.246117:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575780.246118:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1788575780.246122:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575780.246123:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:0.0:1788575780.246126:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575780.246130:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575780.246131:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331092:331092:256:4294967295] 192.168.202.110@tcp LPNI seq info [331049:331049:8:4294967295]
02000000:00000001:0.0:1788575780.246134:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575780.246136:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575780.246141:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1788575780.246143:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788575780.246147:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575780.246155:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575780.246160:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788575780.246172:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:2.0:1788575780.246174:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575780.246175:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575780.246181:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1788575780.246183:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000100:00000001:0.0:1788575780.246192:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575780.246196:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788575780.246197:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1788575780.246202:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:0.0:1788575780.246203:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575780.246212:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12721)
00000800:00000200:2.1:1788575780.246212:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575780.246219:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575780.246226:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575780.246230:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575780.246238:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575780.246244:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575780.246252:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575780.246255:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575780.246260:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575780.246264:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246275:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575780.246279:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575805 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246287:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575780.246289:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575780.246296:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575780.246301:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575780.246305:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307384761
00000100:00000010:2.0:1788575780.246311:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c540.
00000100:00000001:2.0:1788575780.246319:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575780.246340:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307384761
00000100:00000010:2.0:1788575780.246343:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c900.
00000100:00000001:2.0:1788575780.246347:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575780.246351:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575780.246354:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575780.246357:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575780.246361:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575780/real 1788575780]  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575780.246372:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575780.246374:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575780.246377:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575780.246379:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575780.246382:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575780.246386:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575780.246390:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575780.246437:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246449:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575780.246451:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.246453:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575780.246456:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575780.246460:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246467:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575780.246472:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246479:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575780.246481:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.246482:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575780.246483:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.246486:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575780.246498:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575780.246501:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575780.246504:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575780.246507:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575780.246511:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575780.246519:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393235072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575780.246526:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.246528:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575780.246531:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393235072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246536:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575780.246539:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575780.246542:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575780.246545:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575780.246547:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575780.246549:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575780.246551:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575780.246554:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788575780.246559:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575780.246562:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575780.246564:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:2.0:1788575780.246567:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575780.246568:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.246570:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.246580:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575780.246583:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575780.246585:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575780.246587:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575780.246588:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575780.246591:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246598:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575780.246600:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575780.246602:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575780.246604:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575780.246607:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575780.246610:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575780.246614:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575780.246616:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575780.246617:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575780.246620:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575780.246623:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575780.246625:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575780.246629:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575780.246631:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575780.246636:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575780.246639:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393235200, portal 10
00000100:00000001:2.0:1788575780.246642:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575780.246644:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00000040:2.0:1788575780.246651:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246657:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575780.246659:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575780.246660:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575780.246662:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393235200, offset 0
00000400:00000200:2.0:1788575780.246665:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575780.246667:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575780.246671:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575780.246673:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575780.246674:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575780.246676:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575780.246678:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575780.246681:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331093:331093:256:4294967295] 192.168.202.110@tcp LPNI seq info [331050:331050:8:4294967295]
00000400:00000200:2.0:1788575780.246686:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575780.246693:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575780.246697:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575780.246701:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575780.246705:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575780.246708:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575780.246712:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575780.246715:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575780.246719:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575780.246721:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575780.246725:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575780.246728:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575780.246731:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575780.246733:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575780.246736:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575780.246739:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246748:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575780.246750:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575805 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246760:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575780.246761:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575780.246764:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575780.246768:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575780.246772:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575780.246774:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307383847
00000100:00000001:2.0:1788575780.246776:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307384761
00000100:00000001:2.0:1788575780.246779:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307383847
00000100:00000001:2.0:1788575780.246781:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307384761
00000100:00000001:2.0:1788575780.246783:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575780.246785:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575780.246788:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575780.246791:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575780/real 1788575780]  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575780.246801:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575780.246802:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575780.246804:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575780.246807:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575780.246808:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575780.246811:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575780.246814:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575780.246822:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246826:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575780.246827:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.246828:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575780.246829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575780.246831:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246837:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575780.246841:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246848:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575780.246850:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.246852:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575780.246853:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.246854:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575780.246857:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575780.246859:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575780.246863:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575780.246865:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575780.246870:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575780.246877:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575780.246880:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575780.246883:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246889:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575780.246890:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575780.246892:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575780.246894:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575780.246896:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575780.246898:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235328:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575780.246901:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575780.246904:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575780.246905:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575780.246907:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575780.246911:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575780.246912:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575780.246916:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575780.246918:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575780.246922:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575780.246925:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393235328, portal 4
00000100:00000001:2.0:1788575780.246926:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575780.246928:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:2.0:1788575780.246934:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.246941:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575780.246943:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575780.246945:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575780.246947:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393235328, offset 0
00000400:00000200:2.0:1788575780.246952:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575780.246956:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575780.246959:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575780.246962:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575780.246964:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575780.246966:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575780.246969:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575780.246971:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331094:331094:256:4294967295] 192.168.202.110@tcp LPNI seq info [331051:331051:8:4294967295]
00000400:00000200:2.0:1788575780.246975:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575780.246981:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575780.246984:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575780.246987:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575780.246991:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575780.246994:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575780.246997:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575780.247000:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575780.247004:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575780.247007:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575780.247009:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575780.247012:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575780.247014:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575780.247016:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575780.247018:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575780.247021:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.247030:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575780.247032:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575805 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.247038:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575780.247039:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575780.247042:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575780.247046:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575780.247049:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575780.247051:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307383993
00000100:00000001:2.0:1788575780.247053:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307384761
00000100:00000001:2.0:1788575780.247055:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307383993
00000100:00000001:2.0:1788575780.247058:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307384761
00000100:00000001:2.0:1788575780.247060:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575780.247061:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575780.247063:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575780.247066:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575780/real 1788575780]  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575780.247073:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575780.247075:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575780.247077:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575780.247079:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575780.247080:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575780.247083:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575780.247086:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575780.247093:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.247099:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575780.247101:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.247103:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575780.247104:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575780.247115:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.247121:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575780.247124:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.247130:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575780.247136:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.247138:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575780.247139:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.247141:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575780.247143:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575780.247146:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575780.247150:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575780.247152:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575780.247154:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575780.247159:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235328:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575780.247162:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.247163:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575780.247166:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393235200/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.247170:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575780.247171:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575780.247173:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575780.247174:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575780.247175:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575780.247176:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575780.247177:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575780.247179:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788575780.247180:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575780.247181:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575780.247182:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:2.0:1788575780.247184:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575780.247185:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.247186:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575780.247188:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393235328/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575805 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575780.247191:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575780.247192:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575780.247193:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575780.247194:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575780.247194:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575780.247195:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575780.247196:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575780.247197:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:2.0:1788575780.247198:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575780.247199:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575780.247199:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:2.0:1788575780.247201:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575780.247201:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.247202:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575780.247205:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575780.247206:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575780.247207:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575780.247209:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575780.629519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575780.629524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575780.629528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575780.629528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575780.629531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575780.629532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575780.629620:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575780.629623:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575780.629626:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575780.757525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575780.757532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575780.757534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575780.757539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575780.757542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575780.757546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575780.757549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575780.757551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575780.757560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575781.205510:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307384761, at 4307384768
00000100:00000040:1.0:1788575781.205510:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307384761, at 4307384768
00000100:00000010:1.0:1788575781.205526:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00000100:00000010:0.0:1788575781.205528:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00010000:00000001:3.0:1788575781.653525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575781.653531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575781.653537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575781.653538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575781.653542:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575781.653543:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575781.653603:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575781.653607:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575781.653611:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575781.781517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575781.781530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575781.781532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575781.781537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575781.781541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575781.781545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575781.781547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575781.781550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575781.781553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575782.677521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575782.677527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575782.677531:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575782.677532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575782.677535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575782.677536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575782.677593:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575782.677597:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575782.677600:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575782.805511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575782.805516:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575782.805518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575782.805521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575782.805524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575782.805527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575782.805528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575782.805530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575782.805532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575783.701538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575783.701545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575783.701551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575783.701553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575783.701556:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575783.701557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575783.701611:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575783.701626:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575783.701629:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575783.830156:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575783.830164:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575783.830166:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575783.830172:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575783.830175:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575783.830181:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575783.830183:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575783.830186:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575783.830189:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575784.725468:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575784.725474:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575784.725477:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575784.725478:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575784.725480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575784.725481:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575784.725540:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575784.725543:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575784.725547:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575784.853492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575784.853497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575784.853498:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575784.853502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575784.853504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575784.853507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575784.853510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575784.853511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575784.853513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575785.365508:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575785.365527:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575785.365532:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575785.365537:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575785.365541:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575785.365549:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12716
00000400:00000200:1.0:1788575785.365558:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575785.365564:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575785.365567:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575785.365570:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365574:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575785.365577:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365579:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575785.365583:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575785.365585:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575785.365587:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575785.365590:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575785.365593:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575785.365595:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575785.365598:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365600:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575785.365602:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365604:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575785.365607:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575785.365610:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365613:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575785.365616:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575785.365619:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575785.365622:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575785.365624:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575785.365627:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575785.365636:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788575785.365639:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575785.365643:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575785.365645:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575785.365648:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575785.365650:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575785.365654:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575785.365661:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788575785.365667:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575785.365676:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575785.365685:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575785.365689:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575785.365693:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575785.365713:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575785.365716:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575785.365719:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575785.365720:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575785.365722:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575785.365725:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575785.365727:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12716
00000400:00000200:1.0:1788575785.365731:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575785.365735:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575785.365737:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575785.365741:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365743:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575785.365746:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575785.365748:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788575785.365749:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575785.365753:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575785.365754:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:1.0:1788575785.365755:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575785.365758:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1788575785.365759:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575785.365763:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000040:2.0:1788575785.365766:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788575785.365767:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575785.365771:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575785.365778:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575785.365780:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575785.365782:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575785.365783:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575785.365787:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575785.365788:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788575785.365790:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575785.365791:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1788575785.365795:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575785.365795:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575785.365802:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393235456:192.168.202.110@tcp:250:kworker.0
00000100:00080000:1.0:1788575785.365803:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365809:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.365810:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788575785.365813:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575785.365815:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575785.365818:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575785.365819:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1788575785.365821:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575785.365824:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575785.365824:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575785.365826:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575785.365831:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575785.365836:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:2.0:1788575785.365839:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000008:00100000:1.0:1788575785.365842:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575785.365846:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575785.365847:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000001:1.0:1788575785.365848:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575785.365854:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788575785.365858:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000200:2.0:1788575785.365858:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393235456, portal 25
02000000:00000001:1.0:1788575785.365864:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575785.365864:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000001:1.0:1788575785.365869:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.365869:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00000001:1.0:1788575785.365873:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575785.365877:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:2.0:1788575785.365880:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788575785.365881:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.365888:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000010:1.0:1788575785.365894:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000040:2.0:1788575785.365894:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365900:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575785.365901:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575785.365907:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393235456, offset 0
00000100:00100000:1.0:1788575785.365908:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575785.365919:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365924:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575785.365928:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575785.365932:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788575785.365934:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575785.365941:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575785.365947:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575785.365954:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:1.0:1788575785.365955:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575785.365959:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575785.365961:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:1.0:1788575785.365966:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:2.0:1788575785.365968:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:1.0:1788575785.365970:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575785.365973:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575785.365976:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331095:331095:256:4294967295] 192.168.202.110@tcp LPNI seq info [331052:331052:8:4294967295]
00000100:00000001:1.0:1788575785.365978:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575785.365983:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12716
00000400:00000200:2.0:1788575785.365986:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575785.365990:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575785.365994:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575785.365997:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575785.365999:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575785.366003:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575785.366008:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575785.366008:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1788575785.366013:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575785.366017:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.0:1788575785.366018:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:1.0:1788575785.366019:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000010:2.0:1788575785.366025:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000020:00000001:1.0:1788575785.366028:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575785.366031:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575785.366035:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.0:1788575785.366036:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:1.0:1788575785.366039:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575785.366045:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575785.366047:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1788575785.366047:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:1.0:1788575785.366052:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575785.366055:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.1:1788575785.366059:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:1.0:1788575785.366060:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:2.1:1788575785.366066:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00080000:1.0:1788575785.366068:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575785.366072:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575785.366075:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000100:2.0:1788575785.366076:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788575785.366079:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:2.0:1788575785.366079:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
02000000:00000001:1.0:1788575785.366085:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575785.366088:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1788575785.366092:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788575785.366092:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000008:00100000:1.0:1788575785.366097:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575785.366099:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000008:00000001:1.0:1788575785.366102:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575785.366103:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:1.0:1788575785.366105:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.366108:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000010:1.0:1788575785.366110:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000100:00000200:2.0:1788575785.366114:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788575785.366115:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575785.366120:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575785.366122:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575785.366126:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1788575785.366127:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:1.0:1788575785.366128:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575785.366132:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575785.366132:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788575785.366138:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:2.0:1788575785.366141:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575785.366144:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.366144:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575785.366150:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575785.366153:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575785.366157:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575785.366162:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307389881
00000100:00000001:1.0:1788575785.366168:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000010:2.0:1788575785.366170:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c540.
00000100:00000001:1.0:1788575785.366171:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575785.366178:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366178:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1788575785.366188:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575785.366191:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575785.366192:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307389881
00000100:00000040:1.0:1788575785.366197:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:2.0:1788575785.366198:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c900.
00000100:00000001:2.0:1788575785.366204:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:1.0:1788575785.366206:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12726)
00000100:00000001:2.0:1788575785.366208:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575785.366212:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.366214:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575785.366218:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575785/real 1788575785]  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575785.366225:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575785.366227:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575785.366229:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575785.366231:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575785.366232:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575785.366234:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575785.366236:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575785.366241:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366245:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575785.366246:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366248:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575785.366249:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575785.366251:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366255:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575785.366258:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366266:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575785.366268:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366269:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575785.366269:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366271:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575785.366272:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575785.366274:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575785.366277:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575785.366278:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575785.366280:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575785.366285:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393235456:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575785.366288:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366290:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575785.366291:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393235456/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366295:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575785.366297:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575785.366299:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575785.366301:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575785.366302:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575785.366303:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575785.366304:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575785.366306:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:2.0:1788575785.366308:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575785.366309:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575785.366311:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:2.0:1788575785.366314:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575785.366315:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366317:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366332:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575785.366362:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.366365:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575785.366366:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575785.366367:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575785.366370:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366380:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575785.366382:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575785.366384:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575785.366385:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575785.366387:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575785.366390:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235584:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575785.366423:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575785.366425:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575785.366426:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575785.366427:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575785.366429:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575785.366431:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575785.366434:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575785.366436:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575785.366440:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575785.366443:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393235584, portal 10
00000100:00000001:2.0:1788575785.366445:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575785.366446:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:2.0:1788575785.366453:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366459:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575785.366461:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575785.366464:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575785.366466:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393235584, offset 0
00000400:00000200:2.0:1788575785.366470:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575785.366473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575785.366477:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575785.366480:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575785.366482:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575785.366485:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575785.366487:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575785.366490:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331096:331096:256:4294967295] 192.168.202.110@tcp LPNI seq info [331053:331053:8:4294967295]
00000400:00000200:2.0:1788575785.366493:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575785.366500:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575785.366503:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575785.366506:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575785.366510:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575785.366512:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575785.366516:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575785.366519:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575785.366522:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575785.366524:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575785.366527:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575785.366530:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575785.366533:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575785.366535:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575785.366537:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575785.366541:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366548:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575785.366551:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366558:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.366560:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575785.366563:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575785.366566:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575785.366570:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575785.366572:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307388967
00000100:00000001:2.0:1788575785.366575:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307389881
00000100:00000001:2.0:1788575785.366578:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307388967
00000100:00000001:2.0:1788575785.366580:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307389881
00000100:00000001:2.0:1788575785.366582:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575785.366584:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.366586:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575785.366591:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575785/real 1788575785]  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575785.366599:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575785.366601:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575785.366602:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575785.366604:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575785.366606:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575785.366607:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575785.366610:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575785.366617:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366623:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575785.366625:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366626:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575785.366628:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575785.366631:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366637:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575785.366641:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366647:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575785.366650:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366652:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575785.366653:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366655:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575785.366657:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575785.366659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575785.366662:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575785.366664:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575785.366668:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575785.366676:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235584:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575785.366680:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575785.366683:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366689:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575785.366690:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575785.366692:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575785.366694:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575785.366696:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575785.366699:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235712:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575785.366702:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575785.366704:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575785.366705:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575785.366707:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575785.366710:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575785.366712:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575785.366715:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575785.366718:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575785.366721:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575785.366724:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393235712, portal 4
00000100:00000001:2.0:1788575785.366726:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575785.366727:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:2.0:1788575785.366732:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366736:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575785.366737:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575785.366738:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575785.366740:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393235712, offset 0
00000400:00000200:2.0:1788575785.366742:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575785.366745:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575785.366747:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575785.366749:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575785.366751:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575785.366752:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575785.366754:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575785.366756:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331097:331097:256:4294967295] 192.168.202.110@tcp LPNI seq info [331054:331054:8:4294967295]
00000400:00000200:2.0:1788575785.366758:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575785.366763:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575785.366765:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575785.366767:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575785.366769:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575785.366771:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575785.366774:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575785.366775:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575785.366778:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575785.366779:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575785.366781:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575785.366783:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575785.366785:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575785.366786:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575785.366787:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575785.366789:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366794:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575785.366796:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366800:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.366800:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575785.366802:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575785.366804:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575785.366806:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575785.366807:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307389113
00000100:00000001:2.0:1788575785.366808:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307389881
00000100:00000001:2.0:1788575785.366810:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307389113
00000100:00000001:2.0:1788575785.366811:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307389881
00000100:00000001:2.0:1788575785.366812:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575785.366813:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.366814:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575785.366816:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575785/real 1788575785]  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575785.366820:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575785.366821:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575785.366822:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575785.366824:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575785.366824:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575785.366826:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575785.366828:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575785.366835:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366840:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575785.366842:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366843:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575785.366844:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575785.366848:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366854:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575785.366858:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366865:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575785.366867:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366869:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575785.366871:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366872:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575785.366874:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575785.366876:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575785.366879:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575785.366881:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575785.366883:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575785.366888:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235712:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575785.366890:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366891:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575785.366893:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393235584/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366897:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575785.366898:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575785.366899:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575785.366901:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575785.366902:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575785.366902:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575785.366904:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575785.366905:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:2.0:1788575785.366906:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575785.366907:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575785.366908:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:2.0:1788575785.366910:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575785.366910:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366911:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575785.366913:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393235712/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575785.366917:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575785.366917:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575785.366918:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575785.366919:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575785.366920:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575785.366920:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575785.366921:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575785.366922:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:2.0:1788575785.366923:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575785.366924:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575785.366924:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:2.0:1788575785.366925:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575785.366926:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366927:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575785.366929:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575785.366930:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575785.366931:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575785.366932:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575785.749575:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575785.749583:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575785.749590:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575785.749594:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575785.749597:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575785.749601:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575785.749605:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575785.749608:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575785.749610:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575785.749613:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575785.749617:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575785.749619:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575785.749620:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575785.749622:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575785.749623:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575785.749624:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575785.749626:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575785.749628:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575785.749698:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575785.749705:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575785.749709:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575785.877499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575785.877505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575785.877506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575785.877511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575785.877513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575785.877516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575785.877518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575785.877520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575785.877522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575786.325459:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307389881, at 4307389888
00000100:00000010:0.0:1788575786.325471:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c540.
00000100:00000040:1.0:1788575786.325497:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307389881, at 4307389888
00000100:00000010:1.0:1788575786.325512:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c900.
00010000:00000001:3.0:1788575786.773483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575786.773489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575786.773497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575786.773498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575786.773501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575786.773503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575786.773543:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575786.773547:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575786.773554:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575786.901491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575786.901497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575786.901499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575786.901504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575786.901507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575786.901510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575786.901512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575786.901514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575786.901516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575787.797491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575787.797498:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575787.797501:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575787.797503:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575787.797507:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575787.797508:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575787.797510:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575787.797511:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575787.797514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575787.797515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575787.797529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575787.797530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575787.797592:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575787.797595:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575787.797597:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575787.925475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575787.925483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575787.925485:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575787.925491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575787.925495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575787.925499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575787.925502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575787.925504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575787.925508:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788575788.437493:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788575788.437498:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788575788.437502:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788575788.437641:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575788.437655:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:1.0:1788575788.437655:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788575788.437662:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788575788.437665:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575788.821521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575788.821529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575788.821534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575788.821534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575788.821537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575788.821538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575788.821598:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575788.821602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575788.821605:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575788.949519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575788.949525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575788.949527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575788.949531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575788.949533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575788.949537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575788.949539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575788.949541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575788.949549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575789.845477:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575789.845483:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575789.845489:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575789.845490:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575789.845492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575789.845493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575789.845549:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575789.845554:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575789.845557:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575789.973448:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575789.973454:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575789.973456:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575789.973461:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575789.973464:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575789.973468:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575789.973471:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575789.973473:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575789.973476:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575790.485497:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575790.485507:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575790.485510:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575790.485513:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575790.485518:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575790.485524:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12721
00000400:00000200:1.0:1788575790.485533:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575790.485539:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575790.485542:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575790.485545:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485548:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575790.485550:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485552:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575790.485556:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575790.485558:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575790.485561:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575790.485564:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575790.485567:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575790.485569:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575790.485572:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485574:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575790.485575:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485577:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575790.485580:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575790.485584:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485586:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575790.485588:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575790.485591:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575790.485594:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575790.485596:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575790.485598:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575790.485606:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788575790.485609:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575790.485613:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575790.485615:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575790.485617:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575790.485620:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575790.485624:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575790.485630:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788575790.485637:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575790.485645:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575790.485652:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575790.485656:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575790.485658:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575790.485675:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575790.485678:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575790.485680:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575790.485681:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575790.485687:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575790.485690:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575790.485691:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12721
00000400:00000200:1.0:1788575790.485694:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575790.485696:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575790.485698:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575790.485699:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485701:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575790.485702:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485704:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575790.485705:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575790.485706:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575790.485707:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575790.485709:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575790.485710:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575790.485711:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575790.485714:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485715:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575790.485717:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485718:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575790.485720:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575790.485721:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485723:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.485723:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:1.0:1788575790.485724:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575790.485729:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
02000000:00000001:1.0:1788575790.485730:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575790.485732:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1788575790.485733:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575790.485736:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575790.485739:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575790.485741:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575790.485742:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575790.485743:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575790.485747:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788575790.485751:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575790.485754:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575790.485756:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575790.485759:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575790.485760:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575790.485760:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1788575790.485763:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.485763:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575790.485769:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
02000000:00000001:2.0:1788575790.485769:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788575790.485772:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575790.485773:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575790.485777:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575790.485781:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:2.0:1788575790.485784:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393235840:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575790.485792:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1788575790.485793:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575790.485795:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575790.485797:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:1.0:1788575790.485800:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575790.485800:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575790.485805:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575790.485810:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
00000100:00000001:1.0:1788575790.485811:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575790.485815:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:2.0:1788575790.485815:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575790.485820:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575790.485822:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1788575790.485824:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1788575790.485825:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575790.485829:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:2.0:1788575790.485829:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00080000:1.0:1788575790.485832:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12721
00000400:00000200:1.0:1788575790.485835:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575790.485836:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000400:00000200:1.0:1788575790.485837:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575790.485841:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575790.485843:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575790.485844:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393235840, portal 25
00000100:00000001:1.0:1788575790.485845:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575790.485847:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000040:1.0:1788575790.485848:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575790.485849:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345471232 : -130483364080384 : ffff8953796f5500)
00000100:00000001:1.0:1788575790.485851:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575790.485854:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575790.485856:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000040:2.0:1788575790.485857:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788575790.485858:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575790.485860:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575790.485863:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575790.485865:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575790.485867:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485868:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575790.485870:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575790.485871:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575790.485874:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575790.485878:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575790.485880:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1788575790.485881:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575790.485884:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575790.485886:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788575790.485887:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575790.485890:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575790.485891:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000001:1.0:1788575790.485892:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575790.485895:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000200:2.0:1788575790.485896:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393235840, offset 0
00000008:00000001:1.0:1788575790.485897:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575790.485899:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575790.485903:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000400:00000200:2.0:1788575790.485903:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:1.0:1788575790.485906:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575790.485908:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575790.485909:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575790.485909:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575790.485910:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575790.485913:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575790.485913:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:1.0:1788575790.485916:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788575790.485920:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575790.485922:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:1.0:1788575790.485925:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575790.485930:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1788575790.485943:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575790.485944:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:1.0:1788575790.485946:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575790.485950:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:1.0:1788575790.485953:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575790.485957:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575790.485967:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331098:331098:256:4294967295] 192.168.202.110@tcp LPNI seq info [331055:331055:8:4294967295]
00000100:00000001:1.0:1788575790.485968:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575790.485973:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575790.485979:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575790.485980:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788575790.485984:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12731)
00000400:00000200:2.0:1788575790.485994:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575790.486001:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575790.486007:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575790.486014:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575790.486018:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575790.486023:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575790.486026:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1788575790.486029:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575790.486031:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575790.486036:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575790.486040:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575790.486044:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575790.486045:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575790.486048:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575790.486051:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486057:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575790.486060:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486065:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.486066:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575790.486070:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575790.486073:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575790.486074:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307395001
00000100:00000010:2.0:1788575790.486078:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c480.
00000100:00000001:2.0:1788575790.486083:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575790.486092:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307395001
00000100:00000010:2.0:1788575790.486094:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd80.
00000100:00000001:2.0:1788575790.486096:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575790.486099:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575790.486101:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.486103:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575790.486106:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575790/real 1788575790]  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575790.486113:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575790.486114:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575790.486116:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575790.486118:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575790.486119:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575790.486120:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575790.486122:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575790.486128:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486131:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575790.486132:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486134:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575790.486135:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575790.486137:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486141:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575790.486144:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486148:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575790.486150:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486151:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575790.486152:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486153:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575790.486154:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575790.486156:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575790.486158:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575790.486160:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575790.486163:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575790.486168:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393235840:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575790.486171:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486172:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575790.486174:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5500 x1875444393235840/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486177:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575790.486179:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575790.486180:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575790.486182:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575790.486183:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575790.486184:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575790.486185:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575790.486187:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:2.0:1788575790.486189:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575790.486211:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575790.486213:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5500.
00000100:00000001:2.0:1788575790.486216:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575790.486217:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486218:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486224:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575790.486225:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.486226:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575790.486227:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575790.486228:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575790.486231:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486236:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575790.486237:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575790.486238:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575790.486239:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575790.486241:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575790.486242:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575790.486244:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575790.486246:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575790.486247:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575790.486248:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575790.486250:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575790.486251:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575790.486253:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575790.486255:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575790.486257:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575790.486259:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393235968, portal 10
00000100:00000001:2.0:1788575790.486260:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575790.486261:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345469440 : -130483364082176 : ffff8953796f4e00)
00000100:00000040:2.0:1788575790.486265:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486269:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575790.486270:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575790.486271:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575790.486273:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393235968, offset 0
00000400:00000200:2.0:1788575790.486275:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575790.486277:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575790.486279:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575790.486288:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575790.486290:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575790.486291:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575790.486293:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575790.486294:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331099:331099:256:4294967295] 192.168.202.110@tcp LPNI seq info [331056:331056:8:4294967295]
00000400:00000200:2.0:1788575790.486297:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575790.486301:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575790.486304:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575790.486306:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575790.486309:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575790.486310:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575790.486313:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575790.486314:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1788575790.486317:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575790.486318:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575790.486319:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575790.486321:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575790.486323:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575790.486324:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575790.486326:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575790.486328:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486332:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575790.486334:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486338:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.486339:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575790.486341:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575790.486343:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575790.486345:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575790.486346:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307394086
00000100:00000001:2.0:1788575790.486348:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307395001
00000100:00000001:2.0:1788575790.486350:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307394086
00000100:00000001:2.0:1788575790.486352:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307395001
00000100:00000001:2.0:1788575790.486354:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575790.486356:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.486358:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575790.486361:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575790/real 1788575790]  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575790.486375:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575790.486376:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575790.486378:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575790.486380:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575790.486381:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575790.486383:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575790.486386:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575790.486425:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486432:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575790.486433:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486434:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575790.486436:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575790.486439:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486445:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575790.486449:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486456:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575790.486458:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486460:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575790.486461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486463:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575790.486465:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575790.486467:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575790.486470:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575790.486471:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575790.486474:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575790.486482:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393235968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575790.486486:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575790.486489:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486495:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575790.486497:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575790.486499:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575790.486500:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575790.486502:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575790.486505:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575790.486508:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575790.486509:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575790.486511:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575790.486512:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575790.486516:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575790.486518:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575790.486521:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575790.486524:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575790.486527:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575790.486530:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393236096, portal 4
00000100:00000001:2.0:1788575790.486532:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575790.486533:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345481088 : -130483364070528 : ffff8953796f7b80)
00000100:00000040:2.0:1788575790.486539:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486545:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575790.486547:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575790.486549:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575790.486551:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393236096, offset 0
00000400:00000200:2.0:1788575790.486555:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575790.486558:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575790.486560:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575790.486562:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575790.486564:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575790.486565:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575790.486567:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575790.486569:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331100:331100:256:4294967295] 192.168.202.110@tcp LPNI seq info [331057:331057:8:4294967295]
00000400:00000200:2.0:1788575790.486571:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575790.486575:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575790.486578:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575790.486580:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575790.486582:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575790.486584:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788575790.486586:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575790.486588:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1788575790.486590:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575790.486591:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575790.486593:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575790.486595:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575790.486597:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575790.486598:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575790.486600:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575790.486602:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486607:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575790.486608:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486612:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.486613:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575790.486615:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575790.486616:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575790.486618:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575790.486620:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307394233
00000100:00000001:2.0:1788575790.486622:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307395001
00000100:00000001:2.0:1788575790.486623:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307394233
00000100:00000001:2.0:1788575790.486624:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307395001
00000100:00000001:2.0:1788575790.486625:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575790.486626:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.486627:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575790.486629:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575790/real 1788575790]  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575790.486634:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575790.486635:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575790.486636:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575790.486637:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575790.486638:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575790.486639:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575790.486641:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575790.486646:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486649:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575790.486650:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486651:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575790.486651:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575790.486653:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486657:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575790.486659:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486662:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575790.486664:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486665:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575790.486665:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486667:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575790.486668:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575790.486670:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575790.486671:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575790.486672:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575790.486674:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575790.486678:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575790.486680:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486681:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575790.486683:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4e00 x1875444393235968/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486687:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575790.486688:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575790.486689:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575790.486691:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575790.486692:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575790.486692:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575790.486694:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575790.486695:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:2.0:1788575790.486696:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575790.486697:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575790.486698:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4e00.
00000100:00000001:2.0:1788575790.486699:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575790.486700:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486701:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575790.486703:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7b80 x1875444393236096/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575790.486710:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575790.486711:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575790.486712:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575790.486713:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575790.486714:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575790.486714:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575790.486715:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575790.486716:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:2.0:1788575790.486717:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575790.486718:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575790.486718:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7b80.
00000100:00000001:2.0:1788575790.486720:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575790.486720:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486721:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575790.486723:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575790.486724:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575790.486725:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575790.486726:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575790.869492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575790.869497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575790.869501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575790.869502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575790.869504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575790.869505:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575790.869558:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575790.869561:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575790.869564:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575790.997513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575790.997521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575790.997523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575790.997529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575790.997533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575790.997537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575790.997540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575790.997542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575790.997545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575791.445500:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307395001, at 4307395008
00000100:00000040:1.0:1788575791.445500:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307395001, at 4307395008
00000100:00000010:0.0:1788575791.445514:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000100:00000010:1.0:1788575791.445515:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00010000:00000001:3.0:1788575791.893506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575791.893512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575791.893516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575791.893517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575791.893519:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575791.893520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575791.893579:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575791.893583:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575791.893587:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575792.021516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575792.021523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575792.021526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575792.021531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575792.021534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575792.021538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575792.021540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575792.021543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575792.021546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575792.919248:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575792.919253:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575792.919258:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575792.919260:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575792.919263:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575792.919264:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575792.919339:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575792.919343:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575792.919346:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575793.045506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575793.045514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575793.045516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575793.045522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575793.045525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575793.045530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575793.045532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575793.045534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575793.045537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575793.941547:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575793.941557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575793.941564:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575793.941565:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575793.941570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575793.941571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575793.941644:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575793.941650:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575793.941656:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575794.069518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575794.069526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575794.069528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575794.069534:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575794.069537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575794.069541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575794.069544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575794.069547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575794.069549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575794.965513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575794.965524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575794.965530:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575794.965531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575794.965535:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575794.965536:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575794.965598:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575794.965602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575794.965606:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575795.093503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575795.093510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575795.093512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575795.093518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575795.093521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575795.093526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575795.093530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575795.093533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575795.093537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575795.605515:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575795.605522:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575795.605525:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575795.605528:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575795.605532:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575795.605538:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12726
00000400:00000200:2.0:1788575795.605547:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575795.605553:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575795.605555:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575795.605557:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605560:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575795.605562:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605564:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575795.605567:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575795.605568:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575795.605570:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575795.605572:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575795.605574:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575795.605575:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575795.605577:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605578:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575795.605579:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605580:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575795.605582:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575795.605584:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605586:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575795.605588:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575795.605590:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575795.605591:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575795.605593:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575795.605595:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575795.605604:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02d80.
00000020:00000040:2.0:1788575795.605607:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575795.605611:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575795.605612:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575795.605615:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575795.605617:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575795.605621:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575795.605628:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000001:2.0:1788575795.605634:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575795.605643:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575795.605652:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575795.605657:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575795.605661:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009a9ccdb8] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575795.605675:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575795.605678:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575795.605680:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575795.605681:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575795.605682:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575795.605684:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575795.605685:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12726
00000400:00000200:2.0:1788575795.605689:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575795.605691:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575795.605693:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575795.605695:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605697:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575795.605699:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605700:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575795.605702:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1788575795.605702:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1788575795.605703:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575795.605706:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575795.605708:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575795.605708:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1788575795.605712:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575795.605713:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788575795.605714:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575795.605718:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605720:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575795.605723:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:3.0:1788575795.605724:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575795.605730:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575795.605732:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1788575795.605733:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575795.605735:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575795.605737:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605741:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575795.605741:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575795.605745:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575795.605746:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575795.605748:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575795.605751:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1788575795.605752:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393236224:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1788575795.605754:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575795.605758:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575795.605758:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575795.605763:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:2.0:1788575795.605764:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575795.605767:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575795.605767:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575795.605772:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575795.605772:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:2.0:1788575795.605777:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03b80.
02000000:00000010:3.0:1788575795.605778:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
00000020:00000040:2.0:1788575795.605779:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575795.605783:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575795.605784:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575795.605786:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575795.605789:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575795.605793:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:2.0:1788575795.605795:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575795.605798:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1788575795.605799:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000001:2.0:1788575795.605800:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575795.605806:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393236224, portal 25
02000000:00000010:2.0:1788575795.605809:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:3.0:1788575795.605809:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575795.605811:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000001:2.0:1788575795.605816:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575795.605818:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.605825:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575795.605827:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575795.605829:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00100000:2.0:1788575795.605833:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575795.605833:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393236224, offset 0
00000400:00000200:3.0:1788575795.605842:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605850:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575795.605852:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575795.605855:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575795.605860:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1788575795.605864:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037d167ca] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575795.605866:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575795.605872:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1788575795.605876:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575795.605877:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1788575795.605880:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575795.605881:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:2.0:1788575795.605885:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:3.0:1788575795.605888:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331101:331101:256:4294967295] 192.168.202.110@tcp LPNI seq info [331058:331058:8:4294967295]
00000100:00000001:2.0:1788575795.605892:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1788575795.605893:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575795.605896:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575795.605899:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575795.605903:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12726
00000400:00000200:3.0:1788575795.605908:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1788575795.605911:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575795.605919:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:3.0:1788575795.605920:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575795.605922:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000010:3.0:1788575795.605925:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000100:00000040:2.0:1788575795.605926:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605929:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575795.605934:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575795.605935:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575795.605936:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575795.605939:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:3.0:1788575795.605940:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000040:2.0:1788575795.605941:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575795.605944:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575795.605947:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575795.605950:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.1:1788575795.605951:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1788575795.605954:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.1:1788575795.605955:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:2.0:1788575795.605958:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575795.605961:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000100:3.0:1788575795.605964:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575795.605966:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:3.0:1788575795.605967:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000100:00000001:2.0:1788575795.605969:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575795.605972:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575795.605973:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1788575795.605977:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575795.605980:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1788575795.605983:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575795.605985:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
02000000:00000001:2.0:1788575795.605987:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1788575795.605988:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
02000000:00000001:2.0:1788575795.605993:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575795.605993:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575795.605997:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606003:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1788575795.606005:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575795.606007:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575820 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575795.606010:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575795.606014:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:3.0:1788575795.606016:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788575795.606021:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606022:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575795.606025:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606028:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575795.606030:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00000.
00000400:00000200:3.0:1788575795.606032:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575795.606036:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:3.0:1788575795.606037:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307400121
02000000:00000001:2.0:1788575795.606042:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575795.606044:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575795.606045:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575795.606047:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575795.606058:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575795.606058:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
02000000:00000010:2.0:1788575795.606066:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000100:00000001:3.0:1788575795.606067:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575795.606072:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575795.606077:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575795.606079:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307400121
00000100:00000001:2.0:1788575795.606086:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000010:3.0:1788575795.606086:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:2.0:1788575795.606090:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606091:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575795.606094:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575795.606095:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606097:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1788575795.606101:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bf5936f] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575795.606104:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575795/real 1788575795]  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575795.606123:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575795.606127:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:3.0:1788575795.606127:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575795.606130:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:2.0:1788575795.606131:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:3.0:1788575795.606133:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000100:00000040:2.0:1788575795.606137:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12736)
00000400:00000200:3.0:1788575795.606137:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575795.606140:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575795.606143:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575795.606146:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575795.606152:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606157:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575795.606159:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606160:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575795.606162:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575795.606164:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606168:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575795.606171:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606176:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575795.606177:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606178:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575795.606179:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606180:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575795.606181:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575795.606183:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575795.606185:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575795.606187:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575795.606189:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575795.606193:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393236224:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575795.606197:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575795.606200:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393236224/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606204:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575795.606205:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575795.606208:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575795.606209:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575795.606210:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575795.606212:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575795.606213:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575795.606215:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575795.606217:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575795.606219:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575795.606220:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:3.0:1788575795.606222:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575795.606222:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606224:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606229:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575795.606230:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606232:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575795.606233:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575795.606234:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575795.606236:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606240:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575795.606241:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575795.606242:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575795.606243:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575795.606244:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575795.606246:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236352:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575795.606247:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575795.606249:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575795.606250:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575795.606250:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575795.606252:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575795.606253:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575795.606255:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575795.606257:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575795.606259:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575795.606260:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393236352, portal 10
00000100:00000001:3.0:1788575795.606262:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575795.606263:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:3.0:1788575795.606267:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606271:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575795.606272:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575795.606274:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575795.606275:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393236352, offset 0
00000400:00000200:3.0:1788575795.606278:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575795.606280:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575795.606282:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575795.606284:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575795.606286:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575795.606287:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575795.606289:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575795.606291:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331102:331102:256:4294967295] 192.168.202.110@tcp LPNI seq info [331059:331059:8:4294967295]
00000400:00000200:3.0:1788575795.606293:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575795.606297:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575795.606300:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575795.606302:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575795.606304:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575795.606306:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575795.606309:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575795.606310:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575795.606312:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575795.606313:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575795.606315:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575795.606317:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575795.606319:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575795.606320:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575795.606321:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575795.606323:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606331:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575795.606333:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575820 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606337:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606338:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575795.606339:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575795.606341:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575795.606343:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575795.606344:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307399206
00000100:00000001:3.0:1788575795.606346:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307400121
00000100:00000001:3.0:1788575795.606347:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307399206
00000100:00000001:3.0:1788575795.606348:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307400121
00000100:00000001:3.0:1788575795.606350:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575795.606351:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606352:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575795.606354:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575795/real 1788575795]  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575795.606358:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575795.606359:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575795.606359:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575795.606361:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575795.606362:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575795.606363:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575795.606364:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575795.606369:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606372:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575795.606373:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606374:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575795.606375:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575795.606377:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606381:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575795.606383:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606387:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575795.606389:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606390:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575795.606422:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606424:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575795.606426:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575795.606428:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575795.606432:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575795.606434:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575795.606438:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575795.606444:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236352:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575795.606448:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575795.606451:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606456:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575795.606457:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575795.606459:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575795.606461:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575795.606463:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575795.606466:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236480:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575795.606468:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575795.606470:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575795.606471:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575795.606473:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575795.606476:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575795.606478:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575795.606481:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575795.606483:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575795.606486:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575795.606489:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393236480, portal 4
00000100:00000001:3.0:1788575795.606490:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575795.606492:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:3.0:1788575795.606497:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606502:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575795.606504:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575795.606505:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575795.606507:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393236480, offset 0
00000400:00000200:3.0:1788575795.606511:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575795.606514:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575795.606517:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575795.606519:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575795.606521:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575795.606523:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575795.606525:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575795.606528:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331103:331103:256:4294967295] 192.168.202.110@tcp LPNI seq info [331060:331060:8:4294967295]
00000400:00000200:3.0:1788575795.606531:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575795.606537:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575795.606541:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575795.606544:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575795.606547:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575795.606550:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1788575795.606553:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575795.606556:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1788575795.606559:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575795.606561:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575795.606563:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575795.606566:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575795.606568:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575795.606570:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575795.606572:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575795.606575:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575820 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606582:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575795.606585:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575820 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606592:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606593:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575795.606596:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575795.606599:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575795.606602:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575795.606604:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307399353
00000100:00000001:3.0:1788575795.606607:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307400121
00000100:00000001:3.0:1788575795.606608:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307399353
00000100:00000001:3.0:1788575795.606609:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307400121
00000100:00000001:3.0:1788575795.606611:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575795.606612:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606613:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575795.606615:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575795/real 1788575795]  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575795.606621:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575795.606622:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575795.606623:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575795.606624:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575795.606625:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575795.606627:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575795.606628:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575795.606640:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606644:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575795.606645:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606646:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575795.606647:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575795.606649:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606653:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575795.606655:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606659:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575795.606660:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606661:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575795.606662:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606663:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575795.606664:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575795.606665:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575795.606667:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575795.606669:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575795.606672:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575795.606680:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236480:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575795.606682:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606684:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575795.606685:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393236352/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606689:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575795.606690:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575795.606691:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575795.606693:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575795.606694:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575795.606695:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575795.606696:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575795.606697:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575795.606699:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575795.606699:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575795.606700:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:3.0:1788575795.606702:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575795.606703:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606704:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575795.606706:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393236480/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575820 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575795.606709:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575795.606709:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575795.606710:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575795.606711:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575795.606712:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575795.606713:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575795.606713:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575795.606714:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575795.606715:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575795.606716:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575795.606716:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:3.0:1788575795.606718:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575795.606719:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606720:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575795.606722:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575795.606723:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575795.606724:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575795.606725:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575795.989496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575795.989502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575795.989507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575795.989511:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575795.989512:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575795.989515:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575795.989518:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575795.989520:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575795.989522:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575795.989523:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575795.989527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575795.989528:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575795.989529:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575795.989531:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575795.989532:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575795.989533:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575795.989534:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575795.989536:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575795.989592:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575795.989596:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575795.989600:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575796.117508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575796.117515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575796.117517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575796.117523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575796.117527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575796.117530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575796.117533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575796.117535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575796.117538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575796.565504:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307400121, at 4307400128
00000100:00000040:1.0:1788575796.565504:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307400121, at 4307400128
00000100:00000010:0.0:1788575796.565518:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00000100:00000010:1.0:1788575796.565521:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00010000:00000001:3.0:1788575797.013484:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575797.013488:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575797.013492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575797.013493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575797.013495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575797.013496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575797.013551:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575797.013555:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575797.013558:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575797.141456:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575797.141462:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575797.141463:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575797.141467:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575797.141469:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575797.141472:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575797.141474:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575797.141475:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575797.141478:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575798.037502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575798.037510:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575798.037512:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575798.037514:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575798.037519:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575798.037521:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575798.037523:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575798.037526:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575798.037529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575798.037531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575798.037534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575798.037535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575798.037590:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575798.037594:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575798.037597:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575798.165498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575798.165504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575798.165505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575798.165510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575798.165517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575798.165520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575798.165522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575798.165524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575798.165525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575799.061483:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575799.061489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575799.061495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575799.061496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575799.061499:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575799.061501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575799.061544:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575799.061548:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575799.061550:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575799.189501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575799.189508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575799.189510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575799.189516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575799.189520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575799.189524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575799.189526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575799.189528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575799.189530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575800.085496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575800.085503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575800.085508:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575800.085509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575800.085512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575800.085514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575800.085567:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575800.085570:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575800.085579:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575800.213500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575800.213507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575800.213509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575800.213515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575800.213518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575800.213521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575800.213524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575800.213528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575800.213530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575800.725494:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575800.725504:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575800.725507:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575800.725512:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575800.725517:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575800.725524:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12731
00000400:00000200:2.0:1788575800.725532:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575800.725539:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575800.725542:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575800.725545:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575800.725548:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575800.725551:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575800.725553:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575800.725558:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575800.725560:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575800.725562:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575800.725565:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575800.725568:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575800.725570:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575800.725573:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575800.725575:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575800.725577:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575800.725579:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575800.725583:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575800.725586:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575800.725588:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575800.725592:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575800.725595:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575800.725598:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575800.725600:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575800.725602:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575800.725611:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00380.
00000020:00000040:2.0:1788575800.725615:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575800.725618:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575800.725620:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575800.725623:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575800.725625:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575800.725629:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575800.725635:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788575800.725641:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575800.725650:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575800.725659:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575800.725664:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575800.725668:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fc98b80] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575800.725693:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575800.725696:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575800.725699:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575800.725701:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575800.725702:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575800.725705:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575800.725709:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12731
00000400:00000200:2.0:1788575800.725720:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575800.725724:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575800.725726:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575800.725730:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575800.725732:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575800.725733:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575800.725738:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575800.725738:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575800.725743:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575800.725743:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788575800.725747:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575800.725749:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575800.725752:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:3.0:1788575800.725753:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788575800.725755:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575800.725760:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575800.725765:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575800.725773:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575800.725773:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575800.725777:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575800.725777:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575800.725782:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:3.0:1788575800.725784:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1788575800.725785:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575800.725788:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1788575800.725789:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575800.725795:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575800.725796:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575800.725800:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575800.725803:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393236608:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1788575800.725804:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1788575800.725810:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575800.725811:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575800.725814:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575800.725816:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575800.725819:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575800.725820:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575800.725824:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:2.0:1788575800.725825:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575800.725830:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575800.725831:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
00000100:00000001:2.0:1788575800.725832:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575800.725835:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575800.725839:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01880.
00000020:00000040:2.0:1788575800.725844:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:3.0:1788575800.725845:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575800.725850:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575800.725852:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575800.725852:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:2.0:1788575800.725856:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575800.725859:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575800.725861:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575800.725863:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
02000000:00000010:2.0:1788575800.725866:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000100:00000001:2.0:1788575800.725872:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575800.725877:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575800.725878:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393236608, portal 25
00000100:00000001:3.0:1788575800.725883:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575800.725887:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:3.0:1788575800.725898:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575800.725899:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575800.725903:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.725922:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575800.725923:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ab28ae01] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575800.725929:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575800.725933:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000001:2.0:1788575800.725941:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575800.725941:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393236608, offset 0
00000400:00000200:3.0:1788575800.725951:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575800.725953:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575800.725958:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575800.725959:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575800.725961:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1788575800.725966:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575800.725967:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575800.725973:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12731
00000400:00000200:3.0:1788575800.725977:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575800.725984:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575800.725988:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575800.725991:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575800.725994:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575800.725997:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331104:331104:256:4294967295] 192.168.202.110@tcp LPNI seq info [331061:331061:8:4294967295]
00000400:00000200:3.0:1788575800.726002:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575800.726008:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575800.726017:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1788575800.726019:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1788575800.726021:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575800.726028:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575800.726030:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575800.726032:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575800.726037:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:3.0:1788575800.726037:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000100:00000001:2.0:1788575800.726043:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575800.726046:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575800.726049:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:3.0:1788575800.726050:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1788575800.726052:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575800.726056:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575800.726059:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575800.726061:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:3.0:1788575800.726065:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:2.0:1788575800.726066:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575800.726070:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575800.726075:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575800.726079:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575800.726080:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575800.726085:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:3.1:1788575800.726086:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00080000:2.0:1788575800.726091:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575800.726095:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1788575800.726096:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788575800.726097:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:3.0:1788575800.726100:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
02000000:00000001:2.0:1788575800.726102:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575800.726106:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575800.726108:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575800.726109:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00100000:2.0:1788575800.726111:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575800.726117:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575800.726117:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1788575800.726121:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575800.726125:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000100:00000010:2.0:1788575800.726128:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00000.
00000400:00000010:3.0:1788575800.726129:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000020:00000040:2.0:1788575800.726133:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:3.0:1788575800.726136:0:2361:0:(events.c:35:request_out_callback()) Process entered
02000000:00000001:2.0:1788575800.726138:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575800.726141:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575800.726143:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575800.726144:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575800.726148:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575800.726151:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575800.726157:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:3.0:1788575800.726160:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1788575800.726162:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575800.726166:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575800.726171:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575800.726177:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.726179:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575800.726185:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575800.726189:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575800.726193:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307405241
00000100:00000010:3.0:1788575800.726198:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:2.0:1788575800.726207:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575800.726207:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575800.726211:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575800.726217:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bf5936f] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575800.726226:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.726226:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307405241
00000100:00000040:2.0:1788575800.726232:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:3.0:1788575800.726233:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000040:2.0:1788575800.726240:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1788575800.726240:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:2.0:1788575800.726247:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12741)
00000100:00000001:3.0:1788575800.726247:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575800.726253:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.726256:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575800.726262:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575800/real 1788575800]  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575800.726270:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575800.726271:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575800.726273:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575800.726275:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575800.726276:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575800.726278:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575800.726280:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575800.726285:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726292:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575800.726294:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726296:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575800.726299:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575800.726303:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726310:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575800.726314:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726321:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575800.726324:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726326:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575800.726327:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726329:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575800.726331:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575800.726333:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575800.726336:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575800.726338:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575800.726340:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575800.726346:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393236608:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575800.726349:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726350:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575800.726352:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393236608/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726356:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575800.726358:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575800.726360:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575800.726362:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575800.726363:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575800.726364:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575800.726366:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575800.726368:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575800.726370:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575800.726371:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575800.726372:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:3.0:1788575800.726374:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575800.726375:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726377:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726382:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575800.726384:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.726386:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575800.726388:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575800.726389:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575800.726424:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726430:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575800.726431:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575800.726432:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575800.726433:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575800.726435:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575800.726436:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236736:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575800.726438:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575800.726440:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575800.726441:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575800.726442:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575800.726444:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575800.726445:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575800.726448:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575800.726450:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575800.726452:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575800.726454:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393236736, portal 10
00000100:00000001:3.0:1788575800.726455:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575800.726456:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530777728 : -130484178773888 : ffff895348e01880)
00000100:00000040:3.0:1788575800.726460:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726464:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575800.726466:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575800.726467:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575800.726469:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393236736, offset 0
00000400:00000200:3.0:1788575800.726471:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575800.726473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575800.726475:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575800.726477:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575800.726479:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575800.726480:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575800.726482:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575800.726484:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331105:331105:256:4294967295] 192.168.202.110@tcp LPNI seq info [331062:331062:8:4294967295]
00000400:00000200:3.0:1788575800.726486:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575800.726490:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575800.726493:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575800.726495:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575800.726498:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575800.726500:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788575800.726502:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575800.726504:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1788575800.726506:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575800.726507:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575800.726509:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575800.726511:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575800.726513:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575800.726514:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575800.726516:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575800.726518:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726522:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575800.726524:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726532:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.726533:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575800.726535:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575800.726537:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575800.726539:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575800.726540:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307404327
00000100:00000001:3.0:1788575800.726542:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307405241
00000100:00000001:3.0:1788575800.726543:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307404327
00000100:00000001:3.0:1788575800.726544:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307405241
00000100:00000001:3.0:1788575800.726546:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575800.726547:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.726548:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575800.726550:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575800/real 1788575800]  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575800.726554:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575800.726555:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575800.726556:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575800.726557:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575800.726558:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575800.726559:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575800.726561:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575800.726565:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726568:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575800.726569:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726570:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575800.726571:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575800.726573:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726578:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575800.726580:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726584:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575800.726585:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726586:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575800.726587:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726588:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575800.726589:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575800.726590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575800.726592:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575800.726593:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575800.726595:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575800.726599:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236736:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575800.726601:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575800.726603:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726607:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575800.726608:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575800.726609:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575800.726610:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575800.726611:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575800.726613:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575800.726614:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575800.726615:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575800.726616:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575800.726617:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575800.726619:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575800.726621:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575800.726622:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575800.726624:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575800.726626:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575800.726627:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393236864, portal 4
00000100:00000001:3.0:1788575800.726628:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575800.726629:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:3.0:1788575800.726632:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726636:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575800.726637:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575800.726638:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575800.726640:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393236864, offset 0
00000400:00000200:3.0:1788575800.726642:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575800.726643:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575800.726645:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575800.726646:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575800.726648:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575800.726649:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575800.726650:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575800.726652:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331106:331106:256:4294967295] 192.168.202.110@tcp LPNI seq info [331063:331063:8:4294967295]
00000400:00000200:3.0:1788575800.726654:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575800.726658:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575800.726661:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575800.726662:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575800.726665:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575800.726666:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788575800.726668:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575800.726670:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1788575800.726672:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575800.726674:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575800.726675:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575800.726677:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575800.726679:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575800.726680:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575800.726681:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575800.726683:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726687:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575800.726688:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726692:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.726693:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575800.726695:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575800.726696:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575800.726698:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575800.726699:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307404473
00000100:00000001:3.0:1788575800.726700:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307405241
00000100:00000001:3.0:1788575800.726701:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307404473
00000100:00000001:3.0:1788575800.726703:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307405241
00000100:00000001:3.0:1788575800.726704:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575800.726705:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.726706:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575800.726708:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575800/real 1788575800]  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575800.726711:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575800.726712:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575800.726713:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575800.726714:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575800.726715:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575800.726716:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575800.726718:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575800.726723:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726726:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575800.726726:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726727:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575800.726728:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575800.726730:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726733:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575800.726735:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726739:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575800.726740:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726741:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575800.726741:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726742:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575800.726743:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575800.726744:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575800.726746:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575800.726747:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575800.726749:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575800.726753:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393236864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575800.726755:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726756:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575800.726758:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01880 x1875444393236736/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726762:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575800.726763:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575800.726764:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575800.726765:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575800.726766:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575800.726767:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575800.726768:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575800.726769:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575800.726770:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575800.726771:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575800.726772:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01880.
00000100:00000001:3.0:1788575800.726773:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575800.726774:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726775:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575800.726777:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393236864/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575800.726781:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575800.726781:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575800.726782:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575800.726783:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575800.726784:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575800.726784:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575800.726785:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575800.726786:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575800.726787:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575800.726788:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575800.726788:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:3.0:1788575800.726789:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575800.726790:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726791:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575800.726793:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575800.726794:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575800.726795:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575800.726796:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575801.109486:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575801.109492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575801.109498:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575801.109499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575801.109502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575801.109503:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575801.109560:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575801.109564:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575801.109567:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575801.237500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575801.237506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575801.237509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575801.237521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575801.237528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575801.237531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575801.237534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575801.237536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575801.237539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575801.685503:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307405241, at 4307405248
00000100:00000040:1.0:1788575801.685503:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307405241, at 4307405248
00000100:00000010:0.0:1788575801.685517:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000010:1.0:1788575801.685518:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00010000:00000001:3.0:1788575802.133584:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575802.133590:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575802.133594:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575802.133595:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575802.133597:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575802.133598:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575802.133644:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575802.133647:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575802.133650:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575802.261510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575802.261518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575802.261520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575802.261526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575802.261529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575802.261534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575802.261537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575802.261540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575802.261544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575803.157549:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575803.157557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575803.157563:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575803.157564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575803.157568:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575803.157570:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575803.157645:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575803.157650:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575803.157654:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575803.285521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575803.285530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575803.285532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575803.285537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575803.285540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575803.285543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575803.285546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575803.285548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575803.285550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575804.181477:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575804.181484:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575804.181488:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575804.181489:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575804.181492:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575804.181493:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575804.181533:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575804.181536:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575804.181539:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575804.309564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575804.309574:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575804.309577:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575804.309585:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575804.309589:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575804.309594:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575804.309597:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575804.309601:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575804.309604:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575805.205568:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575805.205575:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575805.205580:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575805.205581:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575805.205583:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575805.205584:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575805.205652:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575805.205656:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575805.205660:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575805.333511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575805.333519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575805.333521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575805.333526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575805.333530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575805.333534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575805.333536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575805.333539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575805.333541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575805.845524:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575805.845534:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575805.845537:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575805.845541:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575805.845550:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575805.845556:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12736
00000400:00000200:3.0:1788575805.845565:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575805.845572:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575805.845575:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575805.845577:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575805.845580:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575805.845582:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575805.845583:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575805.845586:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575805.845588:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575805.845590:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575805.845593:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575805.845594:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575805.845596:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575805.845598:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575805.845599:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575805.845600:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575805.845601:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575805.845604:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575805.845606:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575805.845608:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575805.845610:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575805.845613:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575805.845615:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575805.845617:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575805.845619:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575805.845628:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788575805.845631:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575805.845634:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575805.845635:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575805.845638:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575805.845640:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575805.845645:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575805.845653:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575805.845661:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575805.845673:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575805.845683:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575805.845688:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575805.845693:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575805.845716:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575805.845719:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575805.845722:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575805.845723:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575805.845725:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575805.845727:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575805.845730:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12736
00000400:00000200:3.0:1788575805.845735:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575805.845738:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575805.845741:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575805.845743:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575805.845745:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575805.845747:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575805.845748:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575805.845751:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575805.845754:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575805.845755:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575805.845758:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575805.845759:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575805.845760:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575805.845785:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575805.845790:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575805.845796:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575805.845801:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575805.845803:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1788575805.845807:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575805.845808:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575805.845811:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575805.845815:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1788575805.845816:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575805.845820:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575805.845820:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575805.845826:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575805.845827:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575805.845831:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575805.845834:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575805.845836:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1788575805.845840:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393236992:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575805.845842:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575805.845847:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.845848:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575805.845850:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575805.845856:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:3.0:1788575805.845856:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575805.845861:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788575805.845861:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575805.845867:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575805.845868:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:3.0:1788575805.845872:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
02000000:00000010:2.0:1788575805.845876:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
00000020:00000040:3.0:1788575805.845876:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575805.845881:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575805.845882:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575805.845886:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575805.845888:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575805.845889:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575805.845891:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575805.845892:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575805.845897:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
02000000:00000010:3.0:1788575805.845899:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788575805.845904:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575805.845907:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00100000:3.0:1788575805.845912:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575805.845920:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1788575805.845922:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393236992, portal 25
00000100:00000001:3.0:1788575805.845923:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.845928:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1788575805.845931:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.845932:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788575805.845943:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575805.845946:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575805.845950:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575805.845957:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575805.845962:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575805.845968:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00080000:3.0:1788575805.845968:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575805.845971:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:2.0:1788575805.845975:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393236992, offset 0
00000100:00080000:3.0:1788575805.845975:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575805.845982:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575805.845985:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12736
00000400:00000200:2.0:1788575805.845991:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575805.845993:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575805.845998:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575805.846001:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575805.846005:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575805.846007:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575805.846011:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575805.846013:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575805.846016:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575805.846017:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:3.0:1788575805.846021:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575805.846024:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575805.846026:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:2.0:1788575805.846027:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:3.0:1788575805.846029:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575805.846033:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575805.846035:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:3.0:1788575805.846037:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575805.846042:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1788575805.846044:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575805.846049:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575805.846049:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575805.846055:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575805.846060:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331107:331107:256:4294967295] 192.168.202.110@tcp LPNI seq info [331064:331064:8:4294967295]
00000100:00000001:3.0:1788575805.846060:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575805.846066:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575805.846071:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575805.846072:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575805.846077:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575805.846080:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575805.846084:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575805.846088:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1788575805.846089:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575805.846093:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575805.846097:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575805.846101:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575805.846102:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575805.846105:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575805.846111:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000100:00000010:3.0:1788575805.846112:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575805.846117:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575805.846121:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:2.0:1788575805.846123:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:3.0:1788575805.846124:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575805.846127:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575805.846130:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:2.0:1788575805.846131:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
02000000:00000001:3.0:1788575805.846133:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575805.846143:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575805.846147:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
02000000:00000010:3.0:1788575805.846151:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000800:00000100:2.0:1788575805.846152:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575805.846157:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000100:00000001:3.0:1788575805.846158:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575805.846167:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575805.846168:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575805.846178:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1788575805.846181:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575805.846185:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575805.846186:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000100:00000040:3.0:1788575805.846190:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575805.846192:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575805.846200:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:3.0:1788575805.846202:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575805.846205:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1788575805.846207:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575805.846212:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575805.846220:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12747)
00000100:00000001:2.0:1788575805.846225:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575805.846230:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846238:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.846240:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575805.846247:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575805.846252:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575805.846256:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307410361
00000100:00000010:2.0:1788575805.846262:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c0c0.
00000100:00000001:2.0:1788575805.846269:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575805.846275:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307410361
00000100:00000010:2.0:1788575805.846279:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ca20.
00000100:00000001:2.0:1788575805.846282:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575805.846285:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575805.846289:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.846291:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575805.846296:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575805/real 1788575805]  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575805.846307:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575805.846309:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575805.846311:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575805.846314:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575805.846316:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575805.846319:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575805.846323:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575805.846331:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846338:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575805.846340:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.846342:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575805.846344:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575805.846348:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846355:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575805.846359:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846366:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575805.846369:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.846371:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575805.846372:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.846375:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575805.846377:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575805.846380:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575805.846384:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575805.846386:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575805.846391:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575805.846421:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393236992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575805.846426:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.846428:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575805.846431:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393236992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846444:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575805.846446:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575805.846449:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575805.846452:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575805.846454:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575805.846456:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575805.846458:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575805.846461:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575805.846465:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575805.846467:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575805.846469:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575805.846472:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575805.846473:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.846476:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.846484:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575805.846486:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.846488:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575805.846490:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575805.846491:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575805.846495:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846502:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575805.846503:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575805.846505:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575805.846507:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575805.846510:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575805.846513:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575805.846516:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575805.846518:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575805.846520:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575805.846522:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575805.846525:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575805.846527:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575805.846531:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575805.846534:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575805.846537:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575805.846541:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393237120, portal 10
00000100:00000001:2.0:1788575805.846543:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575805.846544:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:2.0:1788575805.846551:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846558:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575805.846560:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575805.846562:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575805.846564:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393237120, offset 0
00000400:00000200:2.0:1788575805.846568:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575805.846571:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575805.846575:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575805.846578:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575805.846581:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575805.846584:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575805.846586:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575805.846589:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331108:331108:256:4294967295] 192.168.202.110@tcp LPNI seq info [331065:331065:8:4294967295]
00000400:00000200:2.0:1788575805.846593:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575805.846600:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575805.846604:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575805.846607:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575805.846611:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575805.846614:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575805.846618:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575805.846621:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575805.846625:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575805.846627:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575805.846630:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575805.846633:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575805.846636:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575805.846638:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575805.846641:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575805.846645:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846652:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575805.846655:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846661:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.846663:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575805.846666:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575805.846670:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575805.846673:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575805.846675:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307409447
00000100:00000001:2.0:1788575805.846678:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307410361
00000100:00000001:2.0:1788575805.846680:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307409447
00000100:00000001:2.0:1788575805.846682:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307410361
00000100:00000001:2.0:1788575805.846684:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575805.846686:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.846689:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575805.846692:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575805/real 1788575805]  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575805.846699:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575805.846701:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575805.846702:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575805.846705:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575805.846706:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575805.846709:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575805.846712:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575805.846720:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846731:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575805.846732:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.846734:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575805.846736:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575805.846739:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846746:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575805.846750:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846756:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575805.846759:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.846760:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575805.846762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.846778:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575805.846781:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575805.846784:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575805.846787:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575805.846789:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575805.846793:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575805.846800:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575805.846804:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575805.846807:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846813:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575805.846815:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575805.846817:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575805.846819:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575805.846821:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575805.846824:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237248:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575805.846827:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575805.846829:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575805.846831:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575805.846832:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575805.846836:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575805.846838:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575805.846841:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575805.846844:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575805.846848:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575805.846850:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393237248, portal 4
00000100:00000001:2.0:1788575805.846852:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575805.846854:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788575805.846860:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846866:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575805.846868:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575805.846870:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575805.846872:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393237248, offset 0
00000400:00000200:2.0:1788575805.846876:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575805.846880:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575805.846883:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575805.846886:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575805.846889:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575805.846891:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575805.846893:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575805.846896:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331109:331109:256:4294967295] 192.168.202.110@tcp LPNI seq info [331066:331066:8:4294967295]
00000400:00000200:2.0:1788575805.846900:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575805.846906:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575805.846911:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575805.846914:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575805.846918:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575805.846921:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575805.846925:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575805.846927:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575805.846931:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575805.846932:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575805.846935:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575805.846939:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575805.846942:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575805.846943:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575805.846946:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575805.846949:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846956:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575805.846959:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.846965:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.846967:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575805.846970:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575805.846973:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575805.846976:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575805.846978:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307409593
00000100:00000001:2.0:1788575805.846981:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307410361
00000100:00000001:2.0:1788575805.846983:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307409593
00000100:00000001:2.0:1788575805.846985:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307410361
00000100:00000001:2.0:1788575805.846987:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575805.846989:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.846991:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575805.846995:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575805/real 1788575805]  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575805.847002:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575805.847004:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575805.847005:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575805.847008:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575805.847009:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575805.847011:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575805.847014:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575805.847022:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.847028:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575805.847029:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.847031:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575805.847033:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575805.847036:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.847042:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575805.847046:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.847053:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575805.847056:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.847057:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575805.847058:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.847060:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575805.847062:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575805.847065:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575805.847068:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575805.847070:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575805.847074:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575805.847081:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237248:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575805.847085:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.847086:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575805.847089:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393237120/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.847095:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575805.847097:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575805.847099:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575805.847101:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575805.847102:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575805.847104:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575805.847106:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575805.847107:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575805.847110:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575805.847111:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575805.847112:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575805.847115:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575805.847116:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.847118:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575805.847121:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393237248/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575805.847127:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575805.847128:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575805.847129:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575805.847132:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575805.847133:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575805.847134:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575805.847136:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575805.847137:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575805.847139:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575805.847140:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575805.847142:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575805.847144:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575805.847145:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.847147:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575805.847150:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575805.847152:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575805.847154:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575805.847155:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575806.229553:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575806.229561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575806.229567:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575806.229571:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575806.229574:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575806.229579:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575806.229582:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575806.229585:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575806.229587:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575806.229589:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575806.229593:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575806.229595:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575806.229596:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575806.229598:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575806.229600:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575806.229601:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575806.229603:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575806.229605:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575806.229678:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575806.229683:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575806.229686:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575806.357593:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575806.357602:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575806.357604:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575806.357610:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575806.357612:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575806.357616:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575806.357619:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575806.357620:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575806.357623:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575806.805539:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307410361, at 4307410368
00000100:00000040:0.0:1788575806.805540:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307410361, at 4307410368
00000100:00000010:1.0:1788575806.805558:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00000100:00000010:0.0:1788575806.805561:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00010000:00000001:3.0:1788575807.253526:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575807.253532:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575807.253536:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575807.253537:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575807.253540:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575807.253541:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575807.253600:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575807.253603:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575807.253606:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575807.381528:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575807.381562:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575807.381566:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575807.381573:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575807.381578:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575807.381584:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575807.381587:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575807.381590:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575807.381593:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575808.277822:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575808.277830:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575808.277833:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575808.277836:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575808.277840:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575808.277842:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575808.277844:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575808.277846:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575808.277850:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575808.277851:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575808.277854:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575808.277855:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575808.278415:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575808.278419:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575808.278422:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575808.405520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575808.405528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575808.405530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575808.405535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575808.405538:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575808.405542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575808.405544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575808.405546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575808.405548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575809.303796:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575809.303803:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575809.303808:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575809.303809:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575809.303813:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575809.303814:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575809.303873:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575809.303876:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575809.303878:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575809.429494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575809.429502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575809.429505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575809.429510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575809.429514:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575809.429517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575809.429519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575809.429522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575809.429525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575810.325509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575810.325515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575810.325521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575810.325522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575810.325525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575810.325526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575810.325581:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575810.325584:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575810.325588:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575810.454734:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575810.454742:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575810.454745:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575810.454750:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575810.454753:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575810.454757:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575810.454760:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575810.454762:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575810.454765:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575810.965538:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575810.965547:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575810.965550:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575810.965554:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575810.965558:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575810.965564:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12742
00000400:00000200:2.0:1788575810.965572:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575810.965578:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575810.965581:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575810.965584:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965587:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575810.965589:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965592:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575810.965596:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575810.965598:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575810.965601:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575810.965604:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575810.965606:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575810.965609:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575810.965612:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965614:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575810.965616:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965618:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575810.965634:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575810.965638:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965640:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575810.965643:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575810.965647:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575810.965649:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575810.965652:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575810.965654:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575810.965663:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03b80.
00000020:00000040:2.0:1788575810.965666:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575810.965670:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575810.965671:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575810.965674:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575810.965677:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575810.965680:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575810.965688:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788575810.965694:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575810.965708:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575810.965718:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575810.965722:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575810.965727:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037d167ca] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575810.965748:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575810.965752:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575810.965755:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575810.965756:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575810.965758:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575810.965761:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575810.965770:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12742
00000400:00000200:2.0:1788575810.965775:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575810.965779:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1788575810.965780:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1788575810.965781:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575810.965785:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1788575810.965786:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965790:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575810.965790:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575810.965794:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965797:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575810.965800:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:3.0:1788575810.965801:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788575810.965802:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575810.965805:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575810.965808:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575810.965811:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575810.965813:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575810.965816:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965818:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575810.965822:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965826:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575810.965830:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575810.965830:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575810.965834:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575810.965835:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575810.965839:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575810.965840:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575810.965842:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575810.965844:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575810.965849:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575810.965849:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575810.965854:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575810.965858:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:3.0:1788575810.965859:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393237376:192.168.202.110@tcp:250:kworker.0
00000008:00100000:2.0:1788575810.965862:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575810.965864:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1788575810.965867:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575810.965870:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575810.965870:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1788575810.965873:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02d80.
02000000:00000001:3.0:1788575810.965875:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788575810.965877:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575810.965879:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1788575810.965881:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575810.965884:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575810.965886:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:3.0:1788575810.965887:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575810.965888:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575810.965891:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575810.965892:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575810.965899:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000400:00000200:3.0:1788575810.965901:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575810.965904:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575810.965908:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00100000:2.0:1788575810.965911:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575810.965916:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575810.965919:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575810.965922:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575810.965926:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009a9ccdb8] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575810.965935:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393237376, portal 25
00000100:00000001:2.0:1788575810.965937:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575810.965939:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1788575810.965941:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1788575810.965943:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00080000:2.0:1788575810.965947:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575810.965950:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575810.965953:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:3.0:1788575810.965954:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575810.965956:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575810.965961:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12742
00000400:00000200:2.0:1788575810.965969:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575810.965973:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:2.0:1788575810.965974:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575810.965978:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575810.965980:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575810.965983:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:3.0:1788575810.965985:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000001:2.0:1788575810.965986:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575810.965990:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:3.0:1788575810.965993:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393237376, offset 0
00000100:00000001:2.0:1788575810.965994:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575810.965999:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575810.966001:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575810.966002:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575810.966005:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:3.0:1788575810.966006:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:2.0:1788575810.966009:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575810.966012:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575810.966017:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575810.966017:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575810.966022:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575810.966024:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575810.966027:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575810.966028:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:2.0:1788575810.966031:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575810.966036:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1788575810.966036:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788575810.966041:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575810.966044:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575810.966044:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:2.0:1788575810.966049:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575810.966051:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:2.0:1788575810.966052:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575810.966056:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575810.966058:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000008:00100000:2.0:1788575810.966060:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575810.966062:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575810.966064:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575810.966067:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03100.
00000020:00000040:2.0:1788575810.966069:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575810.966072:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575810.966073:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575810.966074:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575810.966076:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575810.966077:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575810.966081:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12c00.
00000100:00000001:2.0:1788575810.966084:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575810.966089:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575810.966096:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575810.966098:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575810.966102:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008692924c] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575810.966111:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575810.966115:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575810.966119:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575810.966124:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12752)
00000400:00000200:3.0:1788575810.966124:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331110:331110:256:4294967295] 192.168.202.110@tcp LPNI seq info [331067:331067:8:4294967295]
00000400:00000200:3.0:1788575810.966133:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575810.966145:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575810.966152:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575810.966158:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575810.966164:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575810.966168:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575810.966173:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575810.966176:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575810.966180:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575810.966183:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575810.966189:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575810.966194:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575810.966199:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575810.966202:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575810.966206:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575810.966210:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966219:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575810.966223:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966230:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575810.966232:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575810.966238:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575810.966242:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575810.966245:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307415481
00000100:00000010:3.0:1788575810.966249:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788575810.966256:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575810.966262:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307415481
00000100:00000010:3.0:1788575810.966265:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91da20.
00000100:00000001:3.0:1788575810.966267:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575810.966270:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575810.966273:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575810.966275:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575810.966280:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575810/real 1788575810]  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575810.966289:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575810.966291:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575810.966293:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575810.966296:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575810.966305:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575810.966308:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575810.966312:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575810.966320:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966327:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575810.966329:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.966331:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575810.966334:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575810.966338:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966352:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575810.966359:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966366:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575810.966369:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.966371:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575810.966372:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.966375:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575810.966377:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575810.966380:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575810.966383:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575810.966387:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575810.966410:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575810.966419:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393237376:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575810.966424:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.966426:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575810.966429:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393237376/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966435:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575810.966437:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575810.966440:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575810.966450:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575810.966453:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575810.966454:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575810.966459:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575810.966462:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575810.966466:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575810.966467:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575810.966469:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:3.0:1788575810.966472:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575810.966473:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.966475:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.966482:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575810.966484:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575810.966486:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575810.966488:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575810.966489:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575810.966493:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966499:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575810.966501:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575810.966503:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575810.966505:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575810.966507:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575810.966510:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237504:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575810.966513:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575810.966525:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575810.966527:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575810.966529:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575810.966535:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575810.966536:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575810.966540:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575810.966543:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575810.966546:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575810.966549:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393237504, portal 10
00000100:00000001:3.0:1788575810.966551:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575810.966552:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:3.0:1788575810.966559:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966565:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575810.966567:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575810.966569:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575810.966572:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393237504, offset 0
00000400:00000200:3.0:1788575810.966576:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575810.966580:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575810.966584:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575810.966587:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575810.966589:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575810.966591:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575810.966593:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575810.966596:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331111:331111:256:4294967295] 192.168.202.110@tcp LPNI seq info [331068:331068:8:4294967295]
00000400:00000200:3.0:1788575810.966600:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575810.966606:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575810.966611:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575810.966615:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575810.966619:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575810.966621:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575810.966639:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575810.966642:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575810.966646:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575810.966647:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575810.966650:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575810.966654:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575810.966657:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575810.966659:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575810.966661:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575810.966665:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966673:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575810.966675:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966681:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575810.966683:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575810.966686:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575810.966690:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575810.966693:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575810.966695:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307414567
00000100:00000001:3.0:1788575810.966698:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307415481
00000100:00000001:3.0:1788575810.966700:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307414567
00000100:00000001:3.0:1788575810.966702:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307415481
00000100:00000001:3.0:1788575810.966704:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575810.966706:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575810.966708:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575810.966711:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575810/real 1788575810]  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575810.966725:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575810.966726:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575810.966728:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575810.966731:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575810.966732:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575810.966734:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575810.966738:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575810.966747:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966753:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575810.966754:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.966756:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575810.966757:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575810.966761:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966767:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575810.966771:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966777:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575810.966780:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.966781:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575810.966783:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.966784:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575810.966786:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575810.966789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575810.966792:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575810.966794:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575810.966798:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575810.966806:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237504:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575810.966809:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575810.966812:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966818:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575810.966820:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575810.966822:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575810.966824:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575810.966825:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575810.966828:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237632:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575810.966832:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575810.966833:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575810.966835:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575810.966836:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575810.966840:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575810.966842:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575810.966846:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575810.966848:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575810.966851:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575810.966854:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393237632, portal 4
00000100:00000001:3.0:1788575810.966855:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575810.966857:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:3.0:1788575810.966862:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966868:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575810.966871:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575810.966873:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575810.966875:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393237632, offset 0
00000400:00000200:3.0:1788575810.966880:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575810.966883:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575810.966887:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575810.966889:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575810.966892:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575810.966895:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575810.966897:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575810.966899:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331112:331112:256:4294967295] 192.168.202.110@tcp LPNI seq info [331069:331069:8:4294967295]
00000400:00000200:3.0:1788575810.966904:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575810.966909:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575810.966914:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575810.966917:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575810.966921:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575810.966923:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1788575810.966927:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575810.966929:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1788575810.966933:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575810.966934:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575810.966937:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575810.966940:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575810.966943:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575810.966945:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575810.966947:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575810.966951:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966958:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575810.966961:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.966967:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575810.966968:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575810.966971:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575810.966974:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575810.966977:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575810.966979:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307414713
00000100:00000001:3.0:1788575810.966981:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307415481
00000100:00000001:3.0:1788575810.966983:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307414713
00000100:00000001:3.0:1788575810.966985:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307415481
00000100:00000001:3.0:1788575810.966987:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575810.966989:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575810.966991:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575810.966994:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575810/real 1788575810]  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575810.967002:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575810.967003:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575810.967005:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575810.967007:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575810.967009:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575810.967011:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575810.967014:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575810.967021:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.967028:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575810.967029:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.967031:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575810.967032:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575810.967035:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.967042:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575810.967045:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.967052:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575810.967054:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.967056:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575810.967057:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.967058:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575810.967060:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575810.967062:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575810.967065:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575810.967067:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575810.967071:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575810.967079:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237632:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575810.967082:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.967084:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575810.967087:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393237504/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.967093:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575810.967094:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575810.967096:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575810.967098:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575810.967100:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575810.967101:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575810.967103:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575810.967104:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575810.967107:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575810.967108:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575810.967109:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:3.0:1788575810.967111:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575810.967113:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.967114:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575810.967117:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393237632/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575810.967129:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575810.967130:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575810.967131:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575810.967133:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575810.967134:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575810.967135:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575810.967137:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575810.967139:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575810.967141:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575810.967142:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575810.967144:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:3.0:1788575810.967146:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575810.967147:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.967149:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575810.967152:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575810.967153:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575810.967155:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575810.967156:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575811.349501:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575811.349506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575811.349510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575811.349511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575811.349513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575811.349514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575811.349569:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575811.349573:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575811.349576:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575811.477503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575811.477508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575811.477510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575811.477513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575811.477516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575811.477519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575811.477521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575811.477523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575811.477524:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575811.925511:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307415481, at 4307415488
00000100:00000040:1.0:1788575811.925511:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307415481, at 4307415488
00000100:00000010:1.0:1788575811.925526:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91da20.
00000100:00000010:0.0:1788575811.925529:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00010000:00000001:3.0:1788575812.373503:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575812.373509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575812.373514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575812.373515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575812.373518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575812.373519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575812.373610:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575812.373613:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575812.373615:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575812.501488:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575812.501495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575812.501497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575812.501503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575812.501507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575812.501511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575812.501513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575812.501516:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575812.501519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575813.397522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575813.397528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575813.397533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575813.397534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575813.397538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575813.397539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575813.397662:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575813.397666:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575813.397670:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575813.525496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575813.525504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575813.525506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575813.525512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575813.525515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575813.525519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575813.525521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575813.525523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575813.525526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575814.421514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575814.421520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575814.421525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575814.421555:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575814.421560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575814.421561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575814.421648:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575814.421651:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575814.421654:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575814.549492:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575814.549499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575814.549501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575814.549507:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575814.549511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575814.549515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575814.549517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575814.549519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575814.549551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575815.445481:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575815.445486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575815.445490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575815.445491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575815.445495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575815.445496:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575815.445556:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575815.445558:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575815.445561:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575815.573511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575815.573518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575815.573520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575815.573526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575815.573530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575815.573534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575815.573536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575815.573544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575815.573548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575816.085522:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575816.085533:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575816.085535:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575816.085539:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575816.085543:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575816.085548:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12747
00000400:00000200:3.0:1788575816.085555:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575816.085560:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575816.085562:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575816.085564:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085566:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575816.085568:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085569:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575816.085572:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575816.085573:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575816.085575:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575816.085577:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575816.085579:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575816.085580:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575816.085582:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085583:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575816.085585:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085586:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575816.085589:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575816.085591:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085592:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575816.085595:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575816.085597:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575816.085598:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575816.085600:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575816.085601:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575816.085609:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575816.085611:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575816.085614:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575816.085615:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575816.085617:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575816.085619:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575816.085622:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575816.085627:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575816.085632:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575816.085639:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575816.085646:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575816.085649:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575816.085652:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575816.085668:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575816.085671:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575816.085673:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575816.085674:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575816.085675:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575816.085677:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575816.085679:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12747
00000400:00000200:3.0:1788575816.085681:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575816.085683:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575816.085685:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575816.085688:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085689:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575816.085691:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085692:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575816.085693:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575816.085694:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575816.085695:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575816.085697:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1788575816.085697:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575816.085699:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575816.085702:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575816.085704:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575816.085705:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085708:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575816.085709:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575816.085709:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085713:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575816.085715:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575816.085719:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085720:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575816.085721:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575816.085721:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575816.085726:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575816.085729:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575816.085733:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575816.085737:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788575816.085738:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:3.0:1788575816.085740:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.085742:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575816.085742:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575816.085747:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000020:00000040:3.0:1788575816.085749:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575816.085750:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575816.085750:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575816.085753:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575816.085755:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575816.085755:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575816.085758:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575816.085761:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575816.085762:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575816.085768:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393237760:192.168.202.110@tcp:250:kworker.0
02000000:00000010:3.0:1788575816.085768:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575816.085774:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.085775:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575816.085781:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:3.0:1788575816.085783:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575816.085785:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575816.085791:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575816.085795:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000010:2.0:1788575816.085797:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
00000100:00000001:3.0:1788575816.085798:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575816.085801:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575816.085805:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575816.085812:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575816.085819:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00000001:3.0:1788575816.085821:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575816.085825:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575816.085827:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575816.085828:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:2.0:1788575816.085829:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00080000:3.0:1788575816.085830:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575816.085833:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00000200:2.0:1788575816.085836:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393237760, portal 25
00000100:00080000:3.0:1788575816.085836:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12747
00000100:00000001:2.0:1788575816.085840:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575816.085843:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000400:00000200:3.0:1788575816.085843:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575816.085852:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575816.085852:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575816.085857:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575816.085863:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575816.085866:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575816.085870:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085871:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575816.085875:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000040:3.0:1788575816.085877:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575816.085880:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393237760, offset 0
00000100:00000001:3.0:1788575816.085881:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575816.085886:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575816.085889:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575816.085891:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:2.0:1788575816.085894:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575816.085895:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575816.085899:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575816.085902:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575816.085906:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575816.085907:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575816.085911:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575816.085914:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575816.085916:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788575816.085918:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575816.085922:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575816.085924:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1788575816.085928:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575816.085931:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575816.085933:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575816.085937:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1788575816.085937:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575816.085942:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575816.085945:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331113:331113:256:4294967295] 192.168.202.110@tcp LPNI seq info [331070:331070:8:4294967295]
02000000:00000001:3.0:1788575816.085947:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575816.085951:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575816.085952:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575816.085955:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575816.085959:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:3.0:1788575816.085959:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575816.085963:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575816.085967:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000010:3.0:1788575816.085967:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788575816.085972:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.0:1788575816.085977:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575816.085977:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575816.085980:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575816.085982:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000100:00000001:3.0:1788575816.085983:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575816.085985:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575816.085989:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575816.085993:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575816.085996:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
02000000:00000010:3.0:1788575816.085996:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575816.086002:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575816.086003:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575816.086007:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00100000:3.0:1788575816.086012:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:2.0:1788575816.086015:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575816.086020:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000100:00000001:3.0:1788575816.086026:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575816.086028:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1788575816.086029:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575816.086037:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575816.086038:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575816.086047:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000100:00000001:3.0:1788575816.086049:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575816.086051:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000040:3.0:1788575816.086055:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575816.086057:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:3.0:1788575816.086060:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000200:2.0:1788575816.086066:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575816.086067:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12757)
00000100:00000001:2.0:1788575816.086078:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575816.086081:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086091:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.086092:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575816.086097:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575816.086107:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575816.086109:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307420601
00000100:00000010:2.0:1788575816.086114:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c420.
00000100:00000001:2.0:1788575816.086121:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575816.086126:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307420601
00000100:00000010:2.0:1788575816.086129:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c9c0.
00000100:00000001:2.0:1788575816.086132:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575816.086135:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575816.086138:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.086140:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575816.086145:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575816/real 1788575816]  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575816.086154:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575816.086155:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575816.086157:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575816.086158:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575816.086159:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575816.086161:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575816.086170:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575816.086176:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086180:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575816.086181:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086183:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575816.086184:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575816.086187:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086192:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575816.086195:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086199:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575816.086200:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086201:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575816.086202:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086203:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575816.086205:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575816.086206:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575816.086213:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575816.086214:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575816.086216:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575816.086221:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393237760:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575816.086226:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086227:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575816.086230:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393237760/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086236:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575816.086238:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575816.086240:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575816.086243:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575816.086245:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575816.086246:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575816.086248:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575816.086251:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575816.086254:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575816.086257:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575816.086258:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575816.086262:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575816.086263:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086265:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086271:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575816.086273:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.086274:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575816.086275:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575816.086276:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575816.086280:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086286:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575816.086288:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575816.086290:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575816.086292:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575816.086294:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575816.086296:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575816.086300:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575816.086302:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575816.086303:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575816.086305:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575816.086308:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575816.086310:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575816.086313:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575816.086316:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575816.086320:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575816.086323:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393237888, portal 10
00000100:00000001:2.0:1788575816.086324:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575816.086326:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:2.0:1788575816.086333:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086339:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575816.086342:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575816.086344:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575816.086347:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393237888, offset 0
00000400:00000200:2.0:1788575816.086350:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575816.086354:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575816.086358:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575816.086361:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575816.086364:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575816.086366:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575816.086368:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575816.086371:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331114:331114:256:4294967295] 192.168.202.110@tcp LPNI seq info [331071:331071:8:4294967295]
00000400:00000200:2.0:1788575816.086375:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575816.086381:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575816.086386:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575816.086389:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575816.086412:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575816.086415:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575816.086420:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575816.086422:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575816.086426:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575816.086427:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575816.086430:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575816.086434:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575816.086437:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575816.086439:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575816.086442:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575816.086445:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086453:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575816.086456:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086462:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.086464:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575816.086467:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575816.086471:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575816.086474:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575816.086476:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307419687
00000100:00000001:2.0:1788575816.086479:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307420601
00000100:00000001:2.0:1788575816.086507:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307419687
00000100:00000001:2.0:1788575816.086509:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307420601
00000100:00000001:2.0:1788575816.086512:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575816.086513:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.086515:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575816.086519:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575816/real 1788575816]  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575816.086535:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575816.086536:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575816.086538:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575816.086540:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575816.086542:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575816.086544:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575816.086547:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575816.086554:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086560:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575816.086562:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086564:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575816.086565:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575816.086568:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086574:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575816.086578:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086585:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575816.086587:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086589:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575816.086590:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086592:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575816.086594:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575816.086596:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575816.086599:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575816.086602:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575816.086605:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575816.086613:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393237888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575816.086617:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575816.086620:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086627:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575816.086628:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575816.086630:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575816.086632:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575816.086634:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575816.086637:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575816.086640:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575816.086642:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575816.086643:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575816.086645:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575816.086648:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575816.086650:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575816.086654:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575816.086656:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575816.086660:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575816.086662:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393238016, portal 4
00000100:00000001:2.0:1788575816.086664:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575816.086665:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788575816.086671:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086678:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575816.086680:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575816.086681:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575816.086684:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393238016, offset 0
00000400:00000200:2.0:1788575816.086688:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575816.086691:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575816.086695:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575816.086697:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575816.086700:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575816.086702:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575816.086704:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575816.086707:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331115:331115:256:4294967295] 192.168.202.110@tcp LPNI seq info [331072:331072:8:4294967295]
00000400:00000200:2.0:1788575816.086711:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575816.086717:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575816.086721:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575816.086724:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575816.086728:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575816.086730:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575816.086734:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575816.086737:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575816.086740:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575816.086742:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575816.086745:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575816.086748:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575816.086751:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575816.086752:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575816.086755:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575816.086758:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575841 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086765:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575816.086768:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575841 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086774:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.086775:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575816.086777:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575816.086780:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575816.086783:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575816.086785:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307419833
00000100:00000001:2.0:1788575816.086788:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307420601
00000100:00000001:2.0:1788575816.086790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307419833
00000100:00000001:2.0:1788575816.086792:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307420601
00000100:00000001:2.0:1788575816.086794:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575816.086796:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.086797:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575816.086801:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575816/real 1788575816]  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575816.086808:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575816.086809:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575816.086811:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575816.086813:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575816.086815:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575816.086816:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575816.086820:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575816.086828:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086839:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575816.086841:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086842:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575816.086844:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575816.086847:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086854:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575816.086858:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086865:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575816.086867:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086868:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575816.086869:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086871:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575816.086872:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575816.086875:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575816.086878:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575816.086880:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575816.086884:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575816.086891:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575816.086895:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086896:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575816.086899:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393237888/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086906:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575816.086907:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575816.086909:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575816.086912:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575816.086913:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575816.086915:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575816.086917:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575816.086918:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575816.086920:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575816.086922:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575816.086923:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:2.0:1788575816.086925:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575816.086927:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086928:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575816.086931:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393238016/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575841 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575816.086937:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575816.086938:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575816.086939:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575816.086941:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575816.086942:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575816.086944:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575816.086945:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575816.086947:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575816.086948:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575816.086949:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575816.086951:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575816.086953:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575816.086954:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086956:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575816.086959:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575816.086960:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575816.086962:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575816.086963:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575816.469522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575816.469529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575816.469534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575816.469537:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575816.469539:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575816.469543:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575816.469553:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575816.469555:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575816.469558:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575816.469560:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575816.469564:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575816.469566:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575816.469567:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575816.469568:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575816.469570:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575816.469571:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575816.469573:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575816.469574:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575816.469628:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575816.469631:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575816.469633:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575816.597556:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575816.597563:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575816.597566:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575816.597572:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575816.597575:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575816.597579:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575816.597582:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575816.597585:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575816.597589:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575817.045512:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307420601, at 4307420608
00000100:00000040:0.0:1788575817.045514:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307420601, at 4307420608
00000100:00000010:0.0:1788575817.045527:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00000100:00000010:1.0:1788575817.045531:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00010000:00000001:3.0:1788575817.493523:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575817.493527:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575817.493533:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575817.493534:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575817.493538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575817.493539:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575817.493582:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575817.493585:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575817.493588:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575817.621469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575817.621476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575817.621478:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575817.621483:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575817.621487:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575817.621491:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575817.621493:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575817.621495:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575817.621498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575818.517790:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575818.517796:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575818.517798:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575818.517800:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575818.517803:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575818.517805:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575818.517807:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575818.517808:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575818.517810:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575818.517812:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575818.517814:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575818.517815:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575818.517870:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575818.517873:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575818.517875:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575818.645554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575818.645560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575818.645561:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575818.645565:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575818.645568:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575818.645570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575818.645572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575818.645574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575818.645575:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575819.541538:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575819.541545:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575819.541551:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575819.541553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575819.541556:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575819.541557:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575819.541610:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575819.541614:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575819.541618:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575819.669507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575819.669515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575819.669517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575819.669522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575819.669526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575819.669530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575819.669532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575819.669535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575819.669538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575820.565512:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575820.565518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575820.565524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575820.565525:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575820.565528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575820.565529:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575820.565580:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575820.565584:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575820.565586:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575820.693494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575820.693500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575820.693502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575820.693506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575820.693508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575820.693511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575820.693513:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575820.693515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575820.693523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575821.205514:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575821.205523:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575821.205526:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575821.205529:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575821.205533:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575821.205540:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12752
00000400:00000200:0.0:1788575821.205549:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575821.205554:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575821.205557:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575821.205560:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.205563:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575821.205565:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.205568:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575821.205571:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575821.205573:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575821.205575:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575821.205578:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575821.205581:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575821.205583:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575821.205585:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.205587:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575821.205589:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.205591:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575821.205594:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575821.205596:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575821.205599:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575821.205601:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575821.205604:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575821.205607:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575821.205609:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575821.205611:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575821.205620:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788575821.205623:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575821.205626:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575821.205628:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575821.205631:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575821.205633:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575821.205637:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575821.205644:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788575821.205650:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575821.205659:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575821.205668:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575821.205671:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575821.205676:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575821.205697:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575821.205700:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575821.205704:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575821.205705:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575821.205706:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575821.205709:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575821.205711:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12752
00000400:00000200:0.0:1788575821.205715:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575821.205718:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575821.205721:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575821.205724:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.205726:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575821.205729:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.205731:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575821.205733:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575821.205736:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575821.205736:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1788575821.205739:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575821.205741:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1788575821.205743:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575821.205744:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1788575821.205747:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575821.205752:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575821.205753:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575821.205759:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.205764:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575821.205765:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575821.205768:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575821.205769:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575821.205771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575821.205773:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575821.205774:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1788575821.205777:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575821.205778:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575821.205786:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:2.0:1788575821.205786:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393238144:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575821.205793:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.205794:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575821.205798:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575821.205798:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575821.205801:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575821.205803:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1788575821.205804:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:2.0:1788575821.205807:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788575821.205808:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575821.205811:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575821.205813:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788575821.205818:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:0.0:1788575821.205819:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000010:2.0:1788575821.205822:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000008:00000001:0.0:1788575821.205823:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575821.205827:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575821.205829:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000010:0.0:1788575821.205831:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575821.205835:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000200:2.0:1788575821.205836:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393238144, portal 25
02000000:00000001:0.0:1788575821.205839:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575821.205840:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575821.205842:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
02000000:00000001:0.0:1788575821.205843:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575821.205848:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:2.0:1788575821.205848:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575821.205851:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575821.205855:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.205857:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575821.205860:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000010:0.0:1788575821.205861:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000400:00000010:2.0:1788575821.205863:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000001:0.0:1788575821.205868:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575821.205868:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393238144, offset 0
00000400:00000200:2.0:1788575821.205877:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00100000:0.0:1788575821.205881:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575821.205887:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575821.205895:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575821.205895:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788575821.205899:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575821.205903:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1788575821.205906:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575821.205908:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575821.205913:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575821.205918:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.205925:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575821.205926:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331116:331116:256:4294967295] 192.168.202.110@tcp LPNI seq info [331073:331073:8:4294967295]
00000100:00080000:0.0:1788575821.205930:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575821.205935:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788575821.205937:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575821.205940:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575821.205942:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575821.205949:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575821.205953:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1788575821.205955:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12752
00000400:00000200:0.0:1788575821.205963:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:2.0:1788575821.205967:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575821.205971:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1788575821.205973:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000100:00080000:0.0:1788575821.205975:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575821.205981:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.0:1788575821.205982:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575821.205986:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1788575821.205987:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:0.0:1788575821.205992:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575821.205995:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575821.205997:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.1:1788575821.205999:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1788575821.206002:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575821.206005:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000100:2.0:1788575821.206008:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788575821.206009:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000010:2.0:1788575821.206014:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575821.206020:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:0.0:1788575821.206022:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575821.206026:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575821.206027:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1788575821.206030:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575821.206035:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.206037:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575821.206039:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575821.206041:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575821.206042:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000020:00000040:0.0:1788575821.206044:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:2.0:1788575821.206046:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00080000:0.0:1788575821.206050:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575821.206053:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:0.0:1788575821.206054:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575821.206057:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:2.0:1788575821.206060:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575821.206062:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575821.206067:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575821.206070:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575821.206074:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575821.206080:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575821.206084:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1788575821.206093:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575821.206097:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206098:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575821.206101:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000100:00000001:2.0:1788575821.206102:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000040:0.0:1788575821.206105:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575821.206110:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575821.206111:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206111:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575821.206116:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575821.206119:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575821.206120:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575821.206121:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307425721
02000000:00000010:0.0:1788575821.206127:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788575821.206132:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575821.206132:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c180.
00000100:00100000:0.0:1788575821.206139:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575821.206141:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788575821.206150:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575821.206150:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307425721
00000100:00000001:0.0:1788575821.206155:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575821.206156:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cea0.
00000100:00000001:2.0:1788575821.206160:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:0.0:1788575821.206161:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206163:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575821.206167:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206172:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1788575821.206177:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575821.206178:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575821/real 1788575821]  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575821.206182:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:2.0:1788575821.206189:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1788575821.206190:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000001:2.0:1788575821.206192:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575821.206195:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000100:00000040:0.0:1788575821.206197:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12762)
00000400:00000200:2.0:1788575821.206198:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575821.206201:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575821.206203:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575821.206206:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575821.206212:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206216:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575821.206217:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206218:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575821.206220:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575821.206222:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206226:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575821.206229:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206233:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575821.206235:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206236:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575821.206237:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206238:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575821.206240:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575821.206242:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575821.206244:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575821.206245:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575821.206248:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575821.206252:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393238144:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575821.206255:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206256:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575821.206258:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393238144/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206262:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575821.206264:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575821.206266:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575821.206268:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575821.206269:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575821.206270:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575821.206272:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575821.206274:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788575821.206276:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575821.206277:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575821.206278:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:2.0:1788575821.206280:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575821.206281:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206283:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206288:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575821.206289:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206290:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575821.206291:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575821.206292:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575821.206294:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206299:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575821.206300:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575821.206301:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575821.206302:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575821.206304:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575821.206305:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238272:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575821.206307:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575821.206308:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575821.206309:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575821.206310:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575821.206312:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575821.206313:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575821.206316:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575821.206317:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575821.206320:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575821.206321:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393238272, portal 10
00000100:00000001:2.0:1788575821.206323:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575821.206323:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000040:2.0:1788575821.206328:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206331:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575821.206333:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575821.206334:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575821.206336:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393238272, offset 0
00000400:00000200:2.0:1788575821.206353:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575821.206357:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575821.206361:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575821.206364:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575821.206366:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575821.206368:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575821.206371:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575821.206374:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331117:331117:256:4294967295] 192.168.202.110@tcp LPNI seq info [331074:331074:8:4294967295]
00000400:00000200:2.0:1788575821.206378:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575821.206384:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575821.206388:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575821.206421:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575821.206426:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575821.206429:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575821.206434:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575821.206436:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575821.206440:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575821.206442:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575821.206445:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575821.206448:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575821.206452:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575821.206454:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575821.206456:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575821.206460:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206467:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575821.206470:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206476:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206478:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575821.206481:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575821.206484:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575821.206487:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575821.206489:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307424807
00000100:00000001:2.0:1788575821.206490:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307425721
00000100:00000001:2.0:1788575821.206492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307424807
00000100:00000001:2.0:1788575821.206493:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307425721
00000100:00000001:2.0:1788575821.206494:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575821.206496:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206497:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575821.206500:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575821/real 1788575821]  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575821.206505:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575821.206506:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575821.206507:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575821.206509:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575821.206510:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575821.206511:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575821.206513:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575821.206518:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206521:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575821.206522:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206523:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575821.206524:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575821.206525:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206529:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575821.206531:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206534:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575821.206536:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206537:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575821.206538:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206539:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575821.206540:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575821.206542:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575821.206544:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575821.206545:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575821.206548:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575821.206552:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238272:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575821.206554:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575821.206556:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206559:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575821.206560:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575821.206561:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575821.206562:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575821.206563:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575821.206565:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238400:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575821.206567:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575821.206568:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575821.206569:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575821.206570:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575821.206573:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575821.206574:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575821.206576:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575821.206577:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575821.206579:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575821.206581:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393238400, portal 4
00000100:00000001:2.0:1788575821.206582:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575821.206583:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:2.0:1788575821.206586:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206590:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575821.206592:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575821.206593:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575821.206594:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393238400, offset 0
00000400:00000200:2.0:1788575821.206596:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575821.206599:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575821.206607:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575821.206609:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575821.206610:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575821.206612:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575821.206614:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575821.206616:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331118:331118:256:4294967295] 192.168.202.110@tcp LPNI seq info [331075:331075:8:4294967295]
00000400:00000200:2.0:1788575821.206618:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575821.206622:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575821.206625:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575821.206627:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575821.206629:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575821.206631:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575821.206633:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575821.206635:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575821.206638:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575821.206639:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575821.206641:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575821.206643:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575821.206645:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575821.206646:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575821.206648:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575821.206652:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575846 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206659:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575821.206661:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575846 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206667:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206668:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575821.206672:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575821.206674:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575821.206677:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575821.206679:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307424953
00000100:00000001:2.0:1788575821.206681:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307425721
00000100:00000001:2.0:1788575821.206684:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307424953
00000100:00000001:2.0:1788575821.206686:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307425721
00000100:00000001:2.0:1788575821.206688:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575821.206690:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206692:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575821.206695:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575821/real 1788575821]  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575821.206703:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575821.206704:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575821.206705:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575821.206708:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575821.206709:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575821.206711:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575821.206714:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575821.206722:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206728:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575821.206729:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575821.206733:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575821.206736:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206742:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575821.206744:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206749:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575821.206751:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575821.206752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206753:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575821.206754:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575821.206756:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575821.206758:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575821.206759:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575821.206761:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575821.206765:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238400:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575821.206767:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206768:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575821.206770:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393238272/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206774:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575821.206775:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575821.206776:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575821.206778:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575821.206779:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575821.206779:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575821.206781:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575821.206782:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788575821.206783:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575821.206784:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575821.206785:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:2.0:1788575821.206787:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575821.206787:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206789:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575821.206790:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393238400/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575846 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575821.206794:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575821.206795:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575821.206796:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575821.206797:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575821.206797:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575821.206798:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575821.206799:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575821.206800:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:2.0:1788575821.206802:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575821.206803:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575821.206804:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:2.0:1788575821.206806:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575821.206806:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206808:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575821.206811:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575821.206812:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575821.206813:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575821.206815:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575821.589504:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575821.589512:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575821.589517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575821.589518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575821.589521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575821.589523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575821.589572:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575821.589574:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575821.589577:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575821.717499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575821.717507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575821.717509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575821.717514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575821.717518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575821.717522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575821.717525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575821.717527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575821.717530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575822.165451:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307425721, at 4307425728
00000100:00000010:0.0:1788575822.165463:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000100:00000040:1.0:1788575822.165469:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307425721, at 4307425728
00000100:00000010:1.0:1788575822.165478:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00010000:00000001:3.0:1788575822.613496:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575822.613502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575822.613509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575822.613510:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575822.613514:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575822.613515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575822.613574:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575822.613577:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575822.613580:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575822.741455:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575822.741460:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575822.741462:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575822.741466:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575822.741468:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575822.741471:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575822.741473:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575822.741475:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575822.741477:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575823.637465:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575823.637470:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575823.637474:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575823.637475:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575823.637477:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575823.637478:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575823.637521:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575823.637524:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575823.637527:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575823.765520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575823.765526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575823.765528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575823.765533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575823.765537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575823.765540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575823.765543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575823.765546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575823.765549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575824.661495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575824.661500:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575824.661505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575824.661506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575824.661509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575824.661511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575824.661576:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575824.661579:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575824.661583:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575824.789547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575824.789556:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575824.789558:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575824.789565:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575824.789569:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575824.789574:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575824.789577:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575824.789579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575824.789581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575825.685469:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575825.685473:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575825.685477:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575825.685478:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575825.685480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575825.685481:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575825.685538:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575825.685541:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575825.685545:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575825.813508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575825.813515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575825.813517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575825.813522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575825.813530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575825.813535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575825.813537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575825.813539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575825.813542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1788575826.325509:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:1.0:1788575826.325516:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000100:00080000:3.0:1788575826.325520:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.0:1788575826.325521:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000100:00080000:3.0:1788575826.325528:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575826.325530:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575826.325532:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575826.325535:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575826.325539:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12757
00000400:00000200:3.0:1788575826.325547:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575826.325554:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575826.325557:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575826.325560:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325564:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575826.325567:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325569:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575826.325573:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575826.325575:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575826.325578:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575826.325582:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575826.325584:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575826.325586:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575826.325590:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325592:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575826.325594:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325596:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575826.325599:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575826.325602:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325605:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575826.325608:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575826.325612:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575826.325614:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575826.325616:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575826.325619:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575826.325625:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575826.325629:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575826.325633:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575826.325634:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575826.325637:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575826.325639:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575826.325642:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575826.325647:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575826.325652:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575826.325661:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575826.325668:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575826.325671:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000001:1.0:1788575826.325673:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575826.325674:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000040:3.0:1788575826.325675:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000001:1.0:1788575826.325691:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575826.325692:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575826.325697:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000001:1.0:1788575826.325699:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00080000:3.0:1788575826.325701:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000800:00000001:1.0:1788575826.325704:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575826.325704:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575826.325706:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575826.325709:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575826.325711:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12757
00000400:00000200:3.0:1788575826.325714:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575826.325717:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575826.325719:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575826.325721:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325723:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575826.325725:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325726:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575826.325728:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575826.325729:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575826.325730:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575826.325731:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575826.325733:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575826.325733:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575826.325734:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575826.325738:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575826.325739:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575826.325740:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575826.325743:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575826.325744:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575826.325745:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575826.325748:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575826.325752:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325753:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575826.325755:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575826.325755:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575826.325760:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575826.325764:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575826.325767:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575826.325771:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:3.0:1788575826.325772:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788575826.325774:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575826.325779:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:3.0:1788575826.325781:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575826.325782:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575826.325785:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575826.325787:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575826.325790:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000100:00100000:2.0:1788575826.325794:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393238528:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1788575826.325795:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575826.325801:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575826.325802:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575826.325804:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575826.325807:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575826.325809:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575826.325809:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575826.325813:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575826.325814:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575826.325819:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575826.325821:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
02000000:00000010:2.0:1788575826.325826:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
00000100:00000001:3.0:1788575826.325826:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575826.325831:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575826.325834:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575826.325840:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575826.325842:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575826.325844:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575826.325847:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00000040:3.0:1788575826.325847:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575826.325855:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575826.325856:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00080000:3.0:1788575826.325857:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575826.325861:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575826.325862:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575826.325863:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:2.0:1788575826.325865:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393238528, portal 25
00000100:00000001:3.0:1788575826.325866:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575826.325869:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12757
00000100:00000001:2.0:1788575826.325871:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575826.325875:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000400:00000200:3.0:1788575826.325875:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575826.325880:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575826.325883:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575826.325885:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575826.325888:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325893:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575826.325898:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575826.325901:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575826.325902:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:2.0:1788575826.325907:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:3.0:1788575826.325907:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575826.325910:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000010:2.0:1788575826.325913:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000020:00000001:3.0:1788575826.325914:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575826.325917:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575826.325920:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575826.325921:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575826.325924:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325925:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575826.325927:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575826.325929:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393238528, offset 0
00000100:00000001:3.0:1788575826.325929:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575826.325933:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575826.325936:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575826.325939:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575826.325941:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575826.325942:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575826.325945:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575826.325947:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575826.325949:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575826.325950:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575826.325954:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:3.0:1788575826.325954:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575826.325957:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575826.325960:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000020:00000040:3.0:1788575826.325962:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575826.325964:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:3.0:1788575826.325965:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575826.325967:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575826.325969:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575826.325971:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575826.325972:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1788575826.325973:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575826.325978:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000400:00000200:2.0:1788575826.325979:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575826.325984:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575826.325986:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575826.325992:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00100000:3.0:1788575826.325992:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575826.325998:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331119:331119:256:4294967295] 192.168.202.110@tcp LPNI seq info [331076:331076:8:4294967295]
00000100:00000001:3.0:1788575826.326009:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575826.326011:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575826.326013:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575826.326019:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575826.326026:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1788575826.326031:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575826.326034:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1788575826.326038:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575826.326039:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575826.326045:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12767)
00000800:00000010:2.0:1788575826.326046:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575826.326052:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575826.326057:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575826.326062:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575826.326066:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575826.326070:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575826.326072:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575826.326076:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575826.326081:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575826.326086:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575826.326089:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575826.326092:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575826.326096:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575826.326115:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326121:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575826.326122:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575826.326127:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575826.326132:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575826.326134:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307430841
00000100:00000010:2.0:1788575826.326139:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c5a0.
00000100:00000001:2.0:1788575826.326145:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575826.326151:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307430841
00000100:00000010:2.0:1788575826.326154:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c060.
00000100:00000001:2.0:1788575826.326157:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575826.326160:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575826.326163:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575826.326165:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575826.326169:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575826/real 1788575826]  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575826.326179:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575826.326181:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575826.326183:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575826.326186:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575826.326188:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575826.326190:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575826.326194:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575826.326216:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326229:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575826.326231:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326233:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575826.326235:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575826.326239:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326246:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575826.326250:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326257:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575826.326259:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326261:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575826.326262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326265:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575826.326266:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575826.326269:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575826.326279:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575826.326281:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575826.326286:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575826.326294:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393238528:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575826.326299:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326301:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575826.326304:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393238528/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326311:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575826.326313:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575826.326316:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575826.326319:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575826.326321:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575826.326323:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575826.326325:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575826.326327:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575826.326330:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575826.326332:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575826.326334:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575826.326337:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575826.326339:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326341:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326348:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575826.326350:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575826.326352:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575826.326354:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575826.326355:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575826.326359:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326367:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575826.326368:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575826.326370:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575826.326372:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575826.326374:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575826.326377:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238656:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575826.326380:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575826.326382:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575826.326384:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575826.326386:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575826.326389:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575826.326391:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575826.326423:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575826.326427:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575826.326439:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575826.326442:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393238656, portal 10
00000100:00000001:2.0:1788575826.326444:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575826.326446:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:2.0:1788575826.326452:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326459:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575826.326461:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575826.326464:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575826.326467:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393238656, offset 0
00000400:00000200:2.0:1788575826.326471:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575826.326475:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575826.326478:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575826.326481:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575826.326483:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575826.326485:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575826.326487:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575826.326490:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331120:331120:256:4294967295] 192.168.202.110@tcp LPNI seq info [331077:331077:8:4294967295]
00000400:00000200:2.0:1788575826.326494:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575826.326501:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575826.326505:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575826.326508:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575826.326512:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575826.326515:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575826.326519:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575826.326521:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575826.326525:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575826.326526:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575826.326529:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575826.326532:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575826.326535:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575826.326537:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575826.326539:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575826.326543:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326549:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575826.326552:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326564:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575826.326566:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575826.326569:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575826.326572:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575826.326575:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575826.326577:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307429927
00000100:00000001:2.0:1788575826.326579:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307430841
00000100:00000001:2.0:1788575826.326581:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307429927
00000100:00000001:2.0:1788575826.326583:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307430841
00000100:00000001:2.0:1788575826.326585:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575826.326587:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575826.326589:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575826.326592:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575826/real 1788575826]  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575826.326599:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575826.326601:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575826.326602:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575826.326604:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575826.326606:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575826.326608:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575826.326611:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575826.326618:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326624:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575826.326626:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326627:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575826.326628:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575826.326632:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326638:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575826.326642:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326648:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575826.326650:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326652:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575826.326653:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326655:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575826.326657:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575826.326659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575826.326662:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575826.326665:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575826.326678:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575826.326689:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238656:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575826.326693:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575826.326696:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326702:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575826.326704:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575826.326706:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575826.326707:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575826.326709:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575826.326712:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575826.326715:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575826.326717:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575826.326718:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575826.326720:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575826.326724:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575826.326726:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575826.326729:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575826.326732:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575826.326735:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575826.326738:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393238784, portal 4
00000100:00000001:2.0:1788575826.326740:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575826.326741:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:2.0:1788575826.326746:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326752:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575826.326754:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575826.326756:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575826.326758:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393238784, offset 0
00000400:00000200:2.0:1788575826.326762:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575826.326766:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575826.326769:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575826.326772:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575826.326774:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575826.326776:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575826.326778:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575826.326781:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331121:331121:256:4294967295] 192.168.202.110@tcp LPNI seq info [331078:331078:8:4294967295]
00000400:00000200:2.0:1788575826.326785:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575826.326791:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575826.326795:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575826.326798:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575826.326801:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575826.326804:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575826.326808:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575826.326810:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575826.326814:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575826.326817:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575826.326820:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575826.326823:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575826.326826:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575826.326828:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575826.326830:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575826.326833:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575851 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326840:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575826.326843:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575851 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575826.326850:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575826.326853:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575826.326855:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575826.326858:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575826.326860:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307430073
00000100:00000001:2.0:1788575826.326862:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307430841
00000100:00000001:2.0:1788575826.326864:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307430073
00000100:00000001:2.0:1788575826.326866:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307430841
00000100:00000001:2.0:1788575826.326868:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575826.326870:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575826.326871:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575826.326875:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575826/real 1788575826]  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575826.326882:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575826.326883:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575826.326885:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575826.326887:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575826.326888:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575826.326890:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575826.326893:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575826.326901:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326906:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575826.326907:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326909:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575826.326910:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575826.326913:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326920:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575826.326924:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326930:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575826.326932:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326933:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575826.326934:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326936:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575826.326938:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575826.326940:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575826.326943:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575826.326945:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575826.326949:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575826.326957:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393238784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575826.326960:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326961:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575826.326965:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393238656/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.326971:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575826.326972:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575826.326974:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575826.326976:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575826.326978:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575826.326979:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575826.326981:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575826.326982:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575826.326984:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575826.326985:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575826.326986:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788575826.326989:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575826.326990:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.326991:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575826.326994:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393238784/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575851 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575826.327000:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575826.327001:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575826.327003:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575826.327005:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575826.327006:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575826.327007:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575826.327009:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575826.327010:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575826.327012:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575826.327013:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575826.327014:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788575826.327017:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575826.327018:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.327020:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.327024:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575826.327025:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575826.327027:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575826.327028:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575826.709516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575826.709524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575826.709529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575826.709532:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575826.709534:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575826.709537:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575826.709541:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575826.709543:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575826.709545:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575826.709547:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575826.709550:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575826.709551:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575826.709552:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575826.709553:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575826.709554:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575826.709554:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575826.709555:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575826.709556:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575826.709611:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575826.709614:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575826.709616:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575826.837493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575826.837499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575826.837501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575826.837505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575826.837507:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575826.837510:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.837512:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575826.837514:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575826.837516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575827.285517:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307430841, at 4307430848
00000100:00000040:0.0:1788575827.285518:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307430841, at 4307430848
00000100:00000010:0.0:1788575827.285536:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000100:00000010:1.0:1788575827.285542:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00010000:00000001:3.0:1788575827.733571:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575827.733580:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575827.733588:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575827.733590:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575827.733594:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575827.733595:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575827.733671:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575827.733676:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575827.733680:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575827.861525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575827.861540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575827.861542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575827.861548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575827.861551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575827.861555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575827.861557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575827.861560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575827.861563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575828.757524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575828.757532:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575828.757534:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575828.757537:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575828.757543:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575828.757545:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575828.757547:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575828.757549:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575828.757552:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575828.757554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575828.757557:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575828.757558:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575828.757617:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575828.757620:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575828.757622:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575828.885491:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575828.885497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575828.885499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575828.885502:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575828.885505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575828.885508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575828.885510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575828.885512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575828.885514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575829.781524:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575829.781531:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575829.781537:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575829.781538:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575829.781541:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575829.781542:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575829.781584:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575829.781589:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575829.781592:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575829.909501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575829.909512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575829.909515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575829.909520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575829.909523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575829.909527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575829.909530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575829.909533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575829.909536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575830.805493:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575830.805499:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575830.805505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575830.805506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575830.805509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575830.805511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575830.805560:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575830.805563:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575830.805565:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575830.933496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575830.933502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575830.933504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575830.933509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575830.933512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575830.933515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575830.933517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575830.933519:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575830.933521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575831.445464:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575831.445472:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575831.445475:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575831.445479:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575831.445484:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575831.445491:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12762
00000400:00000200:1.0:1788575831.445500:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575831.445507:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575831.445511:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575831.445514:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575831.445516:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575831.445517:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575831.445519:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575831.445522:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575831.445529:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575831.445531:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575831.445533:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575831.445535:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575831.445536:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575831.445538:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575831.445539:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575831.445541:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575831.445542:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575831.445544:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575831.445546:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575831.445547:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575831.445549:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575831.445552:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575831.445553:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575831.445555:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575831.445557:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575831.445563:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788575831.445565:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575831.445567:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575831.445568:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575831.445571:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575831.445572:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575831.445575:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575831.445581:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788575831.445585:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575831.445592:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575831.445598:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575831.445601:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575831.445604:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575831.445618:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575831.445620:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575831.445622:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575831.445623:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575831.445624:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575831.445626:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575831.445627:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12762
00000400:00000200:1.0:1788575831.445630:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575831.445632:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575831.445634:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575831.445638:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575831.445640:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575831.445642:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575831.445644:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575831.445647:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575831.445647:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:1.0:1788575831.445650:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575831.445652:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1788575831.445654:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575831.445656:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1788575831.445665:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000040:2.0:1788575831.445665:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788575831.445670:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575831.445676:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575831.445683:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1788575831.445684:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575831.445685:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575831.445690:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575831.445690:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575831.445694:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1788575831.445695:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575831.445696:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575831.445701:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00100000:2.0:1788575831.445703:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393238912:192.168.202.110@tcp:250:kworker.0
00000020:00000040:1.0:1788575831.445707:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575831.445708:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575831.445713:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:1.0:1788575831.445714:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575831.445716:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575831.445720:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575831.445720:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1788575831.445724:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:2.0:1788575831.445725:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575831.445727:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575831.445730:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575831.445734:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575831.445736:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575831.445738:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000100:00000001:1.0:1788575831.445740:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:2.0:1788575831.445744:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000008:00100000:1.0:1788575831.445746:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000200:2.0:1788575831.445749:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393238912, portal 25
00000008:00000001:1.0:1788575831.445751:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.445752:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1788575831.445755:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.445755:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345478400 : -130483364073216 : ffff8953796f7100)
00000100:00000040:2.0:1788575831.445762:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1788575831.445764:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000100:00000001:2.0:1788575831.445769:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575831.445771:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575831.445774:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000020:00000040:1.0:1788575831.445776:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000200:2.0:1788575831.445777:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393238912, offset 0
02000000:00000001:1.0:1788575831.445782:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575831.445785:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788575831.445786:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575831.445790:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575831.445792:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575831.445793:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788575831.445795:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575831.445800:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000010:1.0:1788575831.445802:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000400:00000200:2.0:1788575831.445805:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1788575831.445808:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575831.445809:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575831.445814:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575831.445817:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00100000:1.0:1788575831.445819:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575831.445821:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331122:331122:256:4294967295] 192.168.202.110@tcp LPNI seq info [331079:331079:8:4294967295]
00000400:00000200:2.0:1788575831.445830:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575831.445837:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575831.445842:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575831.445842:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575831.445850:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788575831.445851:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1788575831.445854:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000100:00000001:1.0:1788575831.445864:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575831.445867:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1788575831.445873:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575831.445878:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00080000:1.0:1788575831.445885:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:2.1:1788575831.445885:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788575831.445889:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:2.1:1788575831.445891:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00080000:1.0:1788575831.445893:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575831.445898:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000100:2.0:1788575831.445899:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788575831.445903:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12762
00000800:00000010:2.0:1788575831.445905:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:1.0:1788575831.445910:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575831.445913:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1788575831.445915:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575831.445920:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575831.445920:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:1.0:1788575831.445928:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575831.445929:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000100:00000001:1.0:1788575831.445933:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575831.445934:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000040:1.0:1788575831.445938:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575831.445940:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:1.0:1788575831.445942:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:2.0:1788575831.445945:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788575831.445947:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575831.445950:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575831.445953:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575831.445957:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:2.0:1788575831.445957:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575831.445961:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1788575831.445962:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575831.445968:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788575831.445973:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575831.445977:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575831.445977:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788575831.445981:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575831.445983:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575831.445985:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575831.445989:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788575831.445990:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575831.445992:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307435961
00000020:00000040:1.0:1788575831.445996:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000010:2.0:1788575831.445998:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c120.
00000100:00080000:1.0:1788575831.446002:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575831.446005:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446006:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:1.0:1788575831.446008:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575831.446012:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575831.446015:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446016:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307435961
00000100:00000001:1.0:1788575831.446018:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000010:2.0:1788575831.446020:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c780.
00000008:00100000:1.0:1788575831.446022:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575831.446024:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575831.446028:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00000001:1.0:1788575831.446029:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446032:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575831.446034:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446035:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000010:1.0:1788575831.446042:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000100:00100000:2.0:1788575831.446042:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575831/real 1788575831]  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788575831.446048:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000001:2.0:1788575831.446052:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575831.446067:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575831.446069:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575831.446071:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575831.446073:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575831.446074:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575831.446077:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788575831.446080:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575831.446084:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575831.446086:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575831.446088:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1788575831.446093:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575831.446094:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1788575831.446096:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575831.446097:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575831.446100:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1788575831.446101:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575831.446103:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788575831.446108:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:2.0:1788575831.446109:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575831.446114:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575831.446115:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446122:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575831.446125:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1788575831.446126:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575831.446127:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575831.446128:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446131:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575831.446133:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1788575831.446142:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00080000:2.0:1788575831.446142:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:1.0:1788575831.446147:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575831.446149:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575831.446152:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1788575831.446154:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575831.446156:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575831.446176:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393238912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1788575831.446181:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446182:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446186:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575831.446188:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575831.446189:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7100 x1875444393238912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788575831.446198:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575831.446206:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
00000100:00000040:1.0:1788575831.446207:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12772)
02000000:00000001:2.0:1788575831.446208:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575831.446211:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575831.446214:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575831.446215:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575831.446216:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575831.446217:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575831.446219:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:2.0:1788575831.446222:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575831.446223:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575831.446224:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7100.
00000100:00000001:2.0:1788575831.446227:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575831.446227:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446229:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446234:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575831.446235:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446237:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575831.446238:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575831.446239:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575831.446245:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446249:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575831.446250:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575831.446251:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575831.446253:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575831.446254:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575831.446255:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575831.446257:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575831.446259:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575831.446260:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575831.446261:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575831.446263:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575831.446264:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575831.446267:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575831.446269:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575831.446271:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575831.446273:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393239040, portal 10
00000100:00000001:2.0:1788575831.446274:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575831.446275:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345481088 : -130483364070528 : ffff8953796f7b80)
00000100:00000040:2.0:1788575831.446279:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446282:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575831.446284:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575831.446285:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575831.446286:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393239040, offset 0
00000400:00000200:2.0:1788575831.446288:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575831.446291:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575831.446293:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575831.446295:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575831.446297:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575831.446298:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575831.446299:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575831.446301:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331123:331123:256:4294967295] 192.168.202.110@tcp LPNI seq info [331080:331080:8:4294967295]
00000400:00000200:2.0:1788575831.446304:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575831.446308:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575831.446310:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575831.446312:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575831.446315:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575831.446316:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575831.446319:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575831.446321:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575831.446323:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575831.446324:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575831.446326:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575831.446328:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575831.446329:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575831.446331:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575831.446332:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575831.446334:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446339:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575831.446340:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446344:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446345:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575831.446347:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575831.446349:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575831.446351:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575831.446352:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307435046
00000100:00000001:2.0:1788575831.446354:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307435961
00000100:00000001:2.0:1788575831.446355:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307435046
00000100:00000001:2.0:1788575831.446356:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307435961
00000100:00000001:2.0:1788575831.446358:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575831.446359:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446360:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575831.446362:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575831/real 1788575831]  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575831.446366:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575831.446372:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575831.446374:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575831.446375:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575831.446376:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575831.446378:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575831.446380:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575831.446384:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446410:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575831.446411:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446412:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575831.446413:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575831.446416:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446420:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575831.446423:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446426:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575831.446428:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446429:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575831.446429:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446430:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575831.446431:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575831.446433:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575831.446435:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575831.446436:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575831.446438:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575831.446442:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575831.446444:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575831.446446:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446450:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575831.446450:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575831.446452:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575831.446453:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575831.446454:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575831.446456:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239168:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575831.446458:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575831.446459:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575831.446460:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575831.446461:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575831.446463:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575831.446464:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575831.446466:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575831.446468:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575831.446469:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575831.446471:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393239168, portal 4
00000100:00000001:2.0:1788575831.446473:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575831.446473:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345469440 : -130483364082176 : ffff8953796f4e00)
00000100:00000040:2.0:1788575831.446477:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446480:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575831.446482:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575831.446483:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575831.446484:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393239168, offset 0
00000400:00000200:2.0:1788575831.446486:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575831.446488:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575831.446490:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575831.446492:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575831.446493:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575831.446494:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575831.446496:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575831.446498:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331124:331124:256:4294967295] 192.168.202.110@tcp LPNI seq info [331081:331081:8:4294967295]
00000400:00000200:2.0:1788575831.446500:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575831.446504:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575831.446507:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575831.446509:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575831.446511:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575831.446513:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575831.446516:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575831.446517:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575831.446519:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575831.446521:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575831.446523:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575831.446525:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575831.446527:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575831.446528:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575831.446529:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575831.446531:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575856 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446535:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575831.446537:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575856 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446547:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446548:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575831.446550:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575831.446552:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575831.446554:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575831.446555:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307435193
00000100:00000001:2.0:1788575831.446557:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307435961
00000100:00000001:2.0:1788575831.446558:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307435193
00000100:00000001:2.0:1788575831.446560:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307435961
00000100:00000001:2.0:1788575831.446561:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575831.446562:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446563:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575831.446565:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575831/real 1788575831]  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575831.446570:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575831.446570:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575831.446571:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575831.446573:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575831.446574:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575831.446575:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575831.446576:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575831.446581:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446585:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575831.446585:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446586:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575831.446587:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575831.446589:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446592:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575831.446594:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446598:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575831.446599:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446600:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575831.446600:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446601:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575831.446602:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575831.446604:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575831.446605:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575831.446606:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575831.446609:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575831.446613:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239168:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575831.446615:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446616:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575831.446618:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7b80 x1875444393239040/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446621:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575831.446622:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575831.446623:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575831.446624:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575831.446625:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575831.446626:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575831.446627:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575831.446628:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:2.0:1788575831.446629:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575831.446630:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575831.446631:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7b80.
00000100:00000001:2.0:1788575831.446633:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575831.446634:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446636:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575831.446639:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4e00 x1875444393239168/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575856 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575831.446645:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575831.446646:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575831.446647:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575831.446649:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575831.446650:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575831.446651:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575831.446653:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575831.446654:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:2.0:1788575831.446656:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575831.446658:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575831.446659:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4e00.
00000100:00000001:2.0:1788575831.446661:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575831.446663:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446664:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575831.446667:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575831.446669:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575831.446671:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575831.446672:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575831.829502:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575831.829508:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575831.829513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575831.829514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575831.829517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575831.829518:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575831.829576:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575831.829580:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575831.829582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575831.957499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575831.957506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575831.957508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575831.957513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575831.957516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575831.957519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575831.957521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575831.957524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575831.957527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575832.405470:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307435961, at 4307435968
00000100:00000040:1.0:1788575832.405470:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307435961, at 4307435968
00000100:00000010:1.0:1788575832.405483:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000100:00000010:0.0:1788575832.405486:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00010000:00000001:3.0:1788575832.853517:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575832.853524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575832.853529:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575832.853530:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575832.853534:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575832.853535:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575832.853593:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575832.853597:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575832.853601:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575832.981482:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575832.981489:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575832.981492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575832.981496:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575832.981500:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575832.981503:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575832.981506:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575832.981509:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575832.981512:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575833.877558:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575833.877564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575833.877570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575833.877571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575833.877575:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575833.877576:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575833.877943:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575833.877947:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575833.877950:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575834.005464:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575834.005471:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575834.005473:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575834.005478:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575834.005481:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575834.005485:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575834.005487:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575834.005490:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575834.005492:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575834.901505:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575834.901509:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575834.901513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575834.901513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575834.901516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575834.901517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575834.901573:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575834.901577:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575834.901580:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575835.029510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575835.029515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575835.029517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575835.029521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575835.029524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575835.029526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575835.029528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575835.029531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575835.029533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575835.925497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575835.925502:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575835.925506:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575835.925510:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575835.925513:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575835.925516:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575835.925519:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575835.925521:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575835.925523:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575835.925525:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575835.925528:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575835.925530:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575835.925531:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575835.925532:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575835.925534:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575835.925535:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575835.925537:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575835.925538:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575835.925595:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575835.925599:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575835.925602:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575836.053505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575836.053512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575836.053514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575836.053520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575836.053523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575836.053527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575836.053529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575836.053532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575836.053535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575836.565502:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575836.565512:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575836.565514:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575836.565518:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575836.565522:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575836.565529:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12767
00000400:00000200:3.0:1788575836.565537:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575836.565544:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575836.565547:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575836.565550:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565553:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575836.565555:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565557:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575836.565562:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575836.565563:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575836.565566:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575836.565569:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575836.565572:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575836.565573:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575836.565575:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565577:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575836.565578:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565579:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575836.565582:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575836.565584:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565586:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575836.565589:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575836.565592:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575836.565595:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575836.565597:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575836.565600:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575836.565608:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575836.565611:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575836.565615:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575836.565617:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575836.565620:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575836.565623:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575836.565626:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575836.565633:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa000.
00000100:00000001:3.0:1788575836.565638:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575836.565645:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575836.565663:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575836.565667:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575836.565672:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575836.565689:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575836.565692:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575836.565696:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575836.565697:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575836.565699:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575836.565702:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575836.565704:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12767
00000400:00000200:3.0:1788575836.565708:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575836.565711:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575836.565713:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575836.565716:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565719:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575836.565721:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575836.565722:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575836.565726:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575836.565727:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575836.565730:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575836.565731:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788575836.565734:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575836.565738:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575836.565739:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575836.565741:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575836.565745:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575836.565750:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575836.565755:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575836.565758:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575836.565759:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575836.565762:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575836.565763:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565767:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575836.565768:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1788575836.565770:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575836.565773:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1788575836.565774:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575836.565778:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575836.565779:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575836.565783:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1788575836.565785:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393239296:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575836.565789:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575836.565791:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575836.565793:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575836.565795:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1788575836.565797:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575836.565798:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575836.565801:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:3.0:1788575836.565803:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1788575836.565806:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
00000008:00000001:3.0:1788575836.565807:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575836.565809:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575836.565811:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575836.565816:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1788575836.565818:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000400:00000010:2.0:1788575836.565820:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000020:00000040:3.0:1788575836.565822:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000010:2.0:1788575836.565826:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
02000000:00000001:3.0:1788575836.565827:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575836.565831:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575836.565832:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:2.0:1788575836.565833:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393239296, portal 25
02000000:00000001:3.0:1788575836.565834:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575836.565837:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.565838:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575836.565840:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
02000000:00000010:3.0:1788575836.565842:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
00000100:00000040:2.0:1788575836.565846:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575836.565847:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.565856:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:3.0:1788575836.565856:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788575836.565861:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565869:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1788575836.565873:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000001:3.0:1788575836.565873:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575836.565877:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393239296, offset 0
00000100:00000040:3.0:1788575836.565880:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575836.565885:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575836.565894:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575836.565895:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575836.565899:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575836.565901:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:3.0:1788575836.565903:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575836.565905:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1788575836.565907:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1788575836.565907:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575836.565923:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575836.565924:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575836.565929:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1788575836.565930:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12767
00000400:00000200:2.0:1788575836.565936:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575836.565940:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331125:331125:256:4294967295] 192.168.202.110@tcp LPNI seq info [331082:331082:8:4294967295]
00000400:00000200:2.0:1788575836.565946:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575836.565951:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575836.565958:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575836.565961:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1788575836.565963:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575836.565969:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565972:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575836.565974:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.565976:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575836.565977:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.0:1788575836.565979:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575836.565981:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000010:2.0:1788575836.565984:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000020:00000001:3.0:1788575836.565984:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575836.565987:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575836.565990:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.0:1788575836.565994:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788575836.565995:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.0:1788575836.565998:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:3.0:1788575836.566002:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.566007:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788575836.566007:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575836.566011:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:3.0:1788575836.566012:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575836.566019:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000100:2.0:1788575836.566022:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575836.566023:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000010:2.0:1788575836.566026:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000100:00080000:3.0:1788575836.566028:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575836.566031:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575836.566032:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:3.0:1788575836.566032:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575836.566035:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575836.566039:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:3.0:1788575836.566040:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575836.566044:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575836.566046:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575836.566047:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000008:00000001:3.0:1788575836.566048:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575836.566051:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575836.566052:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000010:3.0:1788575836.566054:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000020:00000040:3.0:1788575836.566056:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:2.0:1788575836.566058:0:2361:0:(events.c:35:request_out_callback()) Process entered
02000000:00000001:3.0:1788575836.566058:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575836.566061:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575836.566063:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575836.566065:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1788575836.566067:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575836.566067:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575836.566072:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788575836.566076:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.566089:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00100000:3.0:1788575836.566093:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788575836.566096:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575836.566100:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575836.566102:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575836.566107:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566128:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575836.566128:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575836.566130:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575836.566131:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:3.0:1788575836.566135:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575836.566138:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575836.566139:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12777)
00000400:00000200:2.0:1788575836.566147:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575836.566151:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307441081
00000100:00000010:2.0:1788575836.566156:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cb40.
00000100:00000001:2.0:1788575836.566163:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575836.566176:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307441081
00000100:00000010:2.0:1788575836.566179:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cde0.
00000100:00000001:2.0:1788575836.566183:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575836.566185:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575836.566187:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.566189:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575836.566193:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575836/real 1788575836]  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575836.566200:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575836.566202:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575836.566203:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575836.566206:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575836.566207:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575836.566209:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575836.566212:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575836.566218:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566224:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575836.566226:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566229:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575836.566231:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575836.566235:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566242:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575836.566246:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566254:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575836.566257:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566258:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575836.566260:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566262:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575836.566264:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575836.566266:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575836.566269:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575836.566271:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575836.566274:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575836.566279:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393239296:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575836.566282:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566283:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575836.566285:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393239296/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566288:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575836.566290:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575836.566292:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575836.566294:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575836.566295:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575836.566296:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575836.566298:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575836.566300:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:2.0:1788575836.566302:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575836.566304:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575836.566305:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:2.0:1788575836.566307:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575836.566308:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566309:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566314:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575836.566316:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.566317:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575836.566318:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575836.566319:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575836.566321:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566325:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575836.566326:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575836.566328:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575836.566329:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575836.566330:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575836.566331:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239424:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575836.566333:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575836.566335:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575836.566336:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575836.566337:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575836.566339:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575836.566340:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575836.566343:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575836.566344:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575836.566347:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575836.566348:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393239424, portal 10
00000100:00000001:2.0:1788575836.566350:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575836.566351:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:2.0:1788575836.566355:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566359:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575836.566361:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575836.566362:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575836.566364:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393239424, offset 0
00000400:00000200:2.0:1788575836.566367:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575836.566371:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575836.566374:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575836.566377:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575836.566379:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575836.566381:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575836.566383:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575836.566430:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331126:331126:256:4294967295] 192.168.202.110@tcp LPNI seq info [331083:331083:8:4294967295]
00000400:00000200:2.0:1788575836.566435:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575836.566442:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575836.566447:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575836.566450:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575836.566455:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575836.566458:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575836.566462:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575836.566465:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575836.566469:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575836.566471:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575836.566474:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575836.566477:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575836.566480:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575836.566482:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575836.566485:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575836.566497:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566506:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575836.566510:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566521:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.566523:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575836.566526:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575836.566529:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575836.566533:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575836.566535:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307440167
00000100:00000001:2.0:1788575836.566537:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307441081
00000100:00000001:2.0:1788575836.566539:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307440167
00000100:00000001:2.0:1788575836.566541:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307441081
00000100:00000001:2.0:1788575836.566543:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575836.566545:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.566547:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575836.566552:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575836/real 1788575836]  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575836.566560:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575836.566562:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575836.566564:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575836.566566:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575836.566567:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575836.566570:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575836.566573:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575836.566581:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566587:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575836.566588:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566590:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575836.566591:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575836.566595:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566601:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575836.566603:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566608:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575836.566609:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566610:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575836.566611:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566612:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575836.566614:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575836.566615:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575836.566617:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575836.566618:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575836.566620:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575836.566625:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239424:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575836.566628:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575836.566630:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566633:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575836.566634:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575836.566635:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575836.566637:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575836.566638:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575836.566640:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239552:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575836.566641:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575836.566642:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575836.566644:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575836.566645:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575836.566647:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575836.566648:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575836.566651:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575836.566652:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575836.566655:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575836.566657:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393239552, portal 4
00000100:00000001:2.0:1788575836.566658:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575836.566659:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:2.0:1788575836.566663:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566669:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575836.566672:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575836.566674:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575836.566676:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393239552, offset 0
00000400:00000200:2.0:1788575836.566680:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575836.566683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575836.566687:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575836.566690:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575836.566692:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575836.566694:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575836.566697:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575836.566700:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331127:331127:256:4294967295] 192.168.202.110@tcp LPNI seq info [331084:331084:8:4294967295]
00000400:00000200:2.0:1788575836.566704:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575836.566710:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575836.566714:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575836.566717:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575836.566720:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575836.566723:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575836.566727:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575836.566729:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575836.566731:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575836.566733:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:2.0:1788575836.566735:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575836.566737:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575836.566739:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:2.0:1788575836.566740:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:2.0:1788575836.566741:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575836.566744:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566751:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575836.566754:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566760:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.566761:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575836.566764:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575836.566766:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575836.566769:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575836.566771:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307440313
00000100:00000001:2.0:1788575836.566774:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307441081
00000100:00000001:2.0:1788575836.566775:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307440313
00000100:00000001:2.0:1788575836.566777:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307441081
00000100:00000001:2.0:1788575836.566780:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575836.566782:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.566784:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575836.566787:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575836/real 1788575836]  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575836.566795:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575836.566796:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575836.566797:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575836.566799:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575836.566800:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575836.566801:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575836.566803:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575836.566808:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566811:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575836.566812:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566813:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575836.566814:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575836.566816:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566820:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575836.566822:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566826:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575836.566827:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566828:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575836.566829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566829:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575836.566831:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575836.566832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575836.566834:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575836.566835:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575836.566837:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575836.566841:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239552:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575836.566843:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566844:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575836.566846:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393239424/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566850:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575836.566851:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575836.566852:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575836.566853:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575836.566854:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575836.566855:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575836.566856:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575836.566858:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:2.0:1788575836.566859:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575836.566860:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575836.566861:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788575836.566862:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575836.566863:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566864:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575836.566866:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393239552/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575836.566869:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575836.566870:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575836.566871:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575836.566872:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575836.566872:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575836.566873:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575836.566874:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575836.566875:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575836.566876:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575836.566876:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575836.566877:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:2.0:1788575836.566878:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575836.566879:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566880:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575836.566882:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575836.566884:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575836.566885:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575836.566887:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575836.949486:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575836.949491:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575836.949494:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575836.949495:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575836.949497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575836.949498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575836.949580:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575836.949582:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575836.949584:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575837.077521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575837.077528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575837.077531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575837.077536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575837.077539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575837.077542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575837.077545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575837.077547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575837.077550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575837.525499:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307441081, at 4307441088
00000100:00000040:1.0:1788575837.525499:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307441081, at 4307441088
00000100:00000010:1.0:1788575837.525514:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00000100:00000010:0.0:1788575837.525516:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00010000:00000001:3.0:1788575837.973507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575837.973513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575837.973516:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575837.973517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575837.973520:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575837.973520:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575837.973577:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575837.973580:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575837.973582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575838.101546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575838.101553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575838.101555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575838.101561:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575838.101564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575838.101568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575838.101570:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575838.101574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575838.101578:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575838.997525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575838.997533:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575838.997534:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575838.997538:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575838.997542:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575838.997544:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575838.997546:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575838.997549:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575838.997553:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575838.997554:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575838.997558:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575838.997559:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575838.997657:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575838.997659:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575838.997662:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575839.125626:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575839.125633:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575839.125636:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575839.125641:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575839.125644:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575839.125648:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575839.125650:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575839.125653:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575839.125656:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575840.021787:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575840.021793:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575840.021798:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575840.021799:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575840.021802:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575840.021804:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575840.021870:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575840.021874:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575840.021878:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575840.149534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575840.149540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575840.149543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575840.149548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575840.149551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575840.149554:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575840.149557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575840.149559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575840.149562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575841.045507:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575841.045513:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575841.045518:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575841.045519:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575841.045522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575841.045523:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575841.045589:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575841.045591:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575841.045594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575841.173525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575841.173532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575841.173534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575841.173540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575841.173544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575841.173547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575841.173550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575841.173552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575841.173556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575841.686423:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575841.686433:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575841.686435:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575841.686439:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575841.686443:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575841.686449:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12772
00000400:00000200:1.0:1788575841.686458:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575841.686464:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575841.686467:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575841.686470:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686473:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575841.686475:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686478:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575841.686482:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575841.686484:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575841.686487:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575841.686490:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575841.686492:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575841.686494:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575841.686497:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686499:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575841.686501:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686502:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575841.686505:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575841.686508:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686511:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575841.686513:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575841.686516:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575841.686519:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575841.686521:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575841.686523:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575841.686531:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788575841.686535:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575841.686538:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575841.686540:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575841.686543:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575841.686545:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575841.686548:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575841.686555:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788575841.686561:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575841.686568:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575841.686577:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575841.686582:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575841.686587:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575841.686611:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575841.686614:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575841.686617:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575841.686618:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575841.686620:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575841.686622:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575841.686625:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12772
00000400:00000200:1.0:1788575841.686629:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575841.686631:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575841.686633:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575841.686636:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686638:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575841.686640:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686642:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575841.686644:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575841.686645:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575841.686647:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575841.686649:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575841.686651:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575841.686653:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575841.686656:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686657:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575841.686659:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686661:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575841.686663:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575841.686665:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686667:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575841.686669:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575841.686674:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575841.686676:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575841.686678:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575841.686681:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575841.686684:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575841.686686:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575841.686689:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788575841.686691:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575841.686693:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575841.686694:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575841.686696:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575841.686698:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575841.686699:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575841.686702:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788575841.686705:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575841.686710:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575841.686717:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575841.686719:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575841.686723:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575841.686729:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575841.686731:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575841.686734:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575841.686735:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575841.686737:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575841.686739:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575841.686741:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12772
00000400:00000200:1.0:1788575841.686745:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575841.686747:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575841.686749:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575841.686752:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686754:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575841.686756:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686758:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575841.686760:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575841.686761:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575841.686763:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575841.686764:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575841.686766:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575841.686768:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575841.686799:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686800:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575841.686802:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686804:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575841.686806:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575841.686808:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575841.686810:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575841.686812:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575841.686814:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575841.686816:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575841.686818:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575841.686820:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575841.686822:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575841.686823:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575841.686826:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788575841.686829:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575841.686831:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575841.686832:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575841.686834:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575841.686835:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575841.686837:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575841.686840:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788575841.686843:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575841.686847:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575841.686854:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575841.686856:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575841.686859:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575841.686866:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575841.686869:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575841.686872:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575841.686876:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12782)
00000100:00000001:2.0:1788575841.687660:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575841.687664:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575841.687667:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575841.687674:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575841.687683:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575841.687685:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575841.687688:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575841.687690:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575841.687692:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575841.687696:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393239680:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575841.687700:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575841.687703:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575841.687705:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575841.687707:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575841.687712:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575841.687714:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575841.687719:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575841.687723:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575841.687729:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575841.687735:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393239680, portal 25
00000100:00000001:2.0:1788575841.687738:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575841.687740:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345471232 : -130483364080384 : ffff8953796f5500)
00000100:00000040:2.0:1788575841.687745:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575841.687752:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575841.687755:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575841.687758:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575841.687761:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393239680, offset 0
00000400:00000200:2.0:1788575841.687767:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575841.687787:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575841.687794:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575841.687800:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575841.687804:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575841.687808:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575841.687811:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575841.687814:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331128:331128:256:4294967295] 192.168.202.110@tcp LPNI seq info [331085:331085:8:4294967295]
00000400:00000200:2.0:1788575841.687818:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575841.687828:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575841.687833:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575841.687838:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133000.
00000800:00000200:2.0:1788575841.687844:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575841.687848:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575841.687873:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575841.687878:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575841.687881:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307446203
00000100:00000010:2.0:1788575841.687886:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c960.
00000100:00000001:2.0:1788575841.687893:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575841.687906:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307446203
00000100:00000010:2.0:1788575841.687910:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c8a0.
00000100:00000001:2.0:1788575841.687913:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575841.687916:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575841.687919:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575841.687922:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575841.687926:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575841.687933:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575841.687935:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575841.687937:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575841.687939:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575841.687941:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575841.687945:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575841.687949:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575841.687951:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575841.687952:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575841.687954:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575841.687957:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575841.687959:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575841.687962:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575841.687965:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788575841.687968:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575841.687971:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393239808, portal 10
00000100:00000001:2.0:1788575841.687973:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575841.687974:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:2.0:1788575841.687981:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575841.687987:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575841.687990:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575841.687992:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65660.
00000100:00000200:2.0:1788575841.687994:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393239808, offset 0
00000400:00000200:2.0:1788575841.687998:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575841.688002:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575841.688005:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575841.688008:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575841.688011:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575841.688013:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575841.688015:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575841.688018:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331129:331129:255:4294967295] 192.168.202.110@tcp LPNI seq info [331086:331086:7:4294967295]
00000400:00000200:2.0:1788575841.688022:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575841.688027:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575841.688031:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575841.688034:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133800.
00000800:00000200:2.0:1788575841.688037:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575841.688041:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575841.688043:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575841.688047:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575841.688050:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575841.688051:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307445288
00000100:00000001:2.0:1788575841.688054:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307446203
00000100:00000001:2.0:1788575841.688056:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307445288
00000100:00000001:2.0:1788575841.688058:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307446203
00000100:00000001:2.0:1788575841.688060:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575841.688061:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575841.688064:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575841.688067:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575841.688073:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575841.688075:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575841.688076:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575841.688078:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575841.688080:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575841.688083:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575841.688086:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575841.688088:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575841.688089:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575841.688091:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575841.688093:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575841.688095:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575841.688098:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575841.688101:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575841.688104:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575841.688107:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393239936, portal 4
00000100:00000001:2.0:1788575841.688108:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575841.688110:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:2.0:1788575841.688114:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575841.688120:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575841.688123:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575841.688125:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65aa0.
00000100:00000200:2.0:1788575841.688127:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393239936, offset 0
00000400:00000200:2.0:1788575841.688131:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575841.688134:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575841.688136:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575841.688139:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575841.688141:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575841.688143:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575841.688145:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575841.688148:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331130:331130:254:4294967295] 192.168.202.110@tcp LPNI seq info [331087:331087:6:4294967295]
00000400:00000200:2.0:1788575841.688152:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575841.688157:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575841.688161:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575841.688164:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133500.
00000800:00000200:2.0:1788575841.688168:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575841.688171:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575841.688174:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575841.688176:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575841.688179:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575841.688181:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307445434
00000100:00000001:2.0:1788575841.688183:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307446203
00000100:00000001:2.0:1788575841.688185:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307445434
00000100:00000001:2.0:1788575841.688187:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307446203
00000100:00000001:2.0:1788575841.688189:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575841.688190:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575841.688193:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575841.688195:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575841.688202:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575841.688204:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575841.688206:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575841.688207:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575841.688208:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575841.688210:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575842.069509:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575842.069515:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575842.069521:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575842.069522:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575842.069525:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575842.069526:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575842.069635:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575842.069638:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575842.069641:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575842.197520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575842.197527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575842.197529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575842.197535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575842.197542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575842.197546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575842.197548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575842.197550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575842.197553:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575842.645503:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307446203, at 4307446208
00000100:00000010:1.0:1788575842.645518:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00000100:00000040:0.0:1788575842.646363:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307446203, at 4307446208
00000100:00000010:0.0:1788575842.646371:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00010000:00000001:3.0:1788575843.093500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575843.093506:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575843.093510:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575843.093511:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575843.093513:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575843.093514:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575843.093608:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575843.093610:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575843.093612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575843.221549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575843.221556:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575843.221558:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575843.221563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575843.221567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575843.221570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575843.221573:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575843.221575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575843.221578:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575844.117759:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575844.117765:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575844.117770:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575844.117771:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575844.117775:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575844.117776:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575844.118542:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575844.118546:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575844.118550:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575844.248059:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575844.248066:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575844.248068:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575844.248073:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575844.248076:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575844.248079:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.248082:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575844.248084:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.248087:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788575844.758028:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788575844.758084:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788575844.758095:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575844.758100:0:2352:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788575844.758105:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575844.758110:0:2352:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788575844.758116:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575844.758122:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133000.
00000400:00000200:1.0:1788575844.758128:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575844.758133:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575844.758141:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:1.0:1788575844.758144:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:1.0:1788575844.758150:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575844.758161:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575844.758188:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575844.758193:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575844.758201:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575844.758203:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575844.758213:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575844.758216:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133800.
00000400:00000200:1.0:1788575844.758220:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575844.758224:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575844.758227:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65660
00000400:00000010:1.0:1788575844.758229:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65660.
00000100:00000001:1.0:1788575844.758233:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575844.758238:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575844.758249:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575844.758254:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.758257:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788575844.758262:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575844.758266:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575844.758267:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575844.758272:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000800:00000100:1.0:1788575844.758273:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000100:00100000:2.0:1788575844.758301:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575841/real 1788575844]  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575844.758332:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575844.758334:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575844.758338:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575844.758344:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575844.758347:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575844.758351:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575844.758355:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575844.758361:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.758367:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575844.758368:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.758371:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575844.758372:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575844.758375:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.758379:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575844.758383:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.758387:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575844.758390:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.758429:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575844.758432:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.758434:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575844.758436:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575844.758440:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575844.758444:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575844.758446:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575844.758451:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575844.758459:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393239680:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575844.758464:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575844.758473:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575841/real 1788575844]  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575844.758482:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575844.758483:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575844.758485:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575844.758488:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575844.758490:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575844.758492:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575844.758496:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575844.758503:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.758508:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575844.758509:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.758511:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575844.758512:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575844.758515:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.758520:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575844.758523:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.758528:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575844.758530:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.758532:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575844.758533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.758535:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575844.758537:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575844.758539:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575844.758543:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575844.758545:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575844.758549:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575844.758561:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575844.758566:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575844.758569:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575844.758572:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5500 x1875444393239680/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.758578:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575844.758582:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575844.758586:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575844.758590:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575844.758593:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575844.758595:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575844.758598:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575844.758601:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:2.0:1788575844.758605:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575844.758608:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575844.758609:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5500.
00000100:00000001:2.0:1788575844.758613:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575844.758614:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.758616:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575844.758619:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393239808/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.758626:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575844.758627:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575844.758629:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575844.758641:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575844.758643:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575844.758644:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575844.758647:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575844.758648:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:2.0:1788575844.758651:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575844.758652:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575844.758653:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:2.0:1788575844.758655:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575844.758657:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.758659:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575844.758661:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575844.758662:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575844.758664:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575844.758666:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1788575844.759596:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133500.
00000400:00000200:1.0:1788575844.759603:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575844.759611:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575844.759618:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65aa0
00000400:00000010:1.0:1788575844.759620:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65aa0.
00000100:00000001:1.0:1788575844.759625:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575844.759632:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575844.759662:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575844.759666:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575844.759672:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575844.759674:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575844.761071:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575844.761078:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575844.761081:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575844.761093:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575841/real 1788575844]  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575844.761109:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575844.761111:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575844.761113:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575844.761119:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575844.761122:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575844.761127:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575844.761137:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575844.761148:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.761155:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575844.761157:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.761161:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575844.761163:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575844.761167:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.761174:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575844.761178:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.761185:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575844.761187:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.761188:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575844.761189:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.761191:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575844.761193:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575844.761197:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575844.761201:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575844.761204:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575844.761207:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575844.761215:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393239936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575844.761219:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.761221:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575844.761224:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393239936/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575844.761230:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575844.761233:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575844.761236:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575844.761241:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575844.761243:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575844.761245:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575844.761248:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575844.761251:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:2.0:1788575844.761255:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575844.761257:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575844.761259:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:2.0:1788575844.761263:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575844.761265:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.761267:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575844.761275:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575844.761277:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575844.761279:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575844.761281:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575845.141546:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575845.141553:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575845.141560:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575845.141561:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575845.141564:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575845.141566:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575845.141687:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575845.141693:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575845.141697:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575845.269514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575845.269522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575845.269524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575845.269529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575845.269533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575845.269536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575845.269539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575845.269541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575845.269544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575846.165543:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575846.165549:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575846.165554:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575846.165557:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575846.165559:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575846.165561:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575846.165565:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575846.165567:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575846.165569:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575846.165571:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575846.165575:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575846.165576:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575846.165577:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575846.165579:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575846.165580:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575846.165581:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575846.165582:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575846.165591:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575846.165776:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575846.165783:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575846.165788:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575846.293555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575846.293564:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575846.293566:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575846.293573:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575846.293577:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575846.293582:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.293586:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575846.293589:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.293592:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575846.805499:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575846.805510:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788575846.805513:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575846.805516:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575846.805521:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575846.805529:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12777
00000400:00000200:1.0:1788575846.805538:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575846.805546:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575846.805549:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575846.805552:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805556:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575846.805558:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805560:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575846.805565:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575846.805567:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788575846.805570:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575846.805573:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788575846.805575:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788575846.805577:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788575846.805581:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805583:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575846.805585:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805598:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575846.805602:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788575846.805606:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805609:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575846.805612:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575846.805616:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575846.805619:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575846.805621:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788575846.805624:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575846.805631:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788575846.805644:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788575846.805648:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575846.805650:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575846.805653:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575846.805655:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575846.805658:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575846.805665:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788575846.805672:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575846.805680:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575846.805690:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575846.805694:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575846.805699:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575846.805858:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575846.805862:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575846.805865:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788575846.805866:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575846.805868:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575846.805871:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575846.805874:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12777
00000400:00000200:1.0:1788575846.805878:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575846.805881:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575846.805883:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575846.805886:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805888:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575846.805890:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805892:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575846.805894:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575846.805895:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788575846.805897:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575846.805899:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788575846.805900:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788575846.805902:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788575846.805905:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805907:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575846.805908:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805910:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575846.805912:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788575846.805915:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575846.805917:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575846.805919:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575846.805924:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575846.805926:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575846.805928:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575846.805932:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788575846.805935:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575846.805937:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575846.805941:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
00000020:00000040:1.0:1788575846.805943:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788575846.805946:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575846.805947:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575846.805949:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575846.805950:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575846.805952:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575846.805956:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788575846.805960:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575846.805966:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575846.805974:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575846.805976:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575846.805979:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575846.805987:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788575846.805989:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788575846.805993:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788575846.805994:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788575846.805996:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788575846.805998:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788575846.806001:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12777
00000400:00000200:1.0:1788575846.806005:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575846.806007:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788575846.806009:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788575846.806012:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.806014:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575846.806016:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.806018:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788575846.806020:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788575846.806021:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788575846.806023:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788575846.806025:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788575846.806027:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788575846.806029:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788575846.806032:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.806034:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788575846.806036:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788575846.806038:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788575846.806040:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788575846.806043:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788575846.806045:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788575846.806047:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788575846.806050:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788575846.806051:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575846.806053:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788575846.806055:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788575846.806057:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575846.806059:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788575846.806062:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788575846.806064:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788575846.806067:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788575846.806068:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575846.806069:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788575846.806071:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788575846.806073:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788575846.806076:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788575846.806079:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575846.806083:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788575846.806090:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788575846.806092:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575846.806095:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575846.806103:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575846.806106:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788575846.806109:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788575846.806115:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12788)
00000100:00000001:2.0:1788575846.808040:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575846.808052:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575846.808056:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575846.808073:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808092:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575846.808095:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575846.808099:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575846.808104:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575846.808107:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575846.808112:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393240064:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575846.808116:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575846.808122:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575846.808125:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575846.808128:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575846.808141:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575846.808143:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575846.808152:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575846.808157:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575846.808167:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575846.808177:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393240064, portal 25
00000100:00000001:2.0:1788575846.808179:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575846.808182:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00000040:2.0:1788575846.808189:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808197:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575846.808200:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575846.808204:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575846.808209:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393240064, offset 0
00000400:00000200:2.0:1788575846.808217:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575846.808227:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575846.808237:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575846.808245:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575846.808250:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575846.808254:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575846.808257:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575846.808262:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331131:331131:256:4294967295] 192.168.202.110@tcp LPNI seq info [331088:331088:8:4294967295]
00000400:00000200:2.0:1788575846.808267:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575846.808279:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575846.808286:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575846.808292:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575846.808299:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575846.808304:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575846.808310:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575846.808314:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575846.808319:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575846.808322:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575846.808329:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575846.808335:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575846.808342:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575846.808344:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575846.808349:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575846.808354:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808366:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575846.808370:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808378:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575846.808380:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575846.808387:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575846.808455:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575846.808459:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307451324
00000100:00000010:2.0:1788575846.808465:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c360.
00000100:00000001:2.0:1788575846.808474:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575846.808483:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307451324
00000100:00000010:2.0:1788575846.808487:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd20.
00000100:00000001:2.0:1788575846.808490:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575846.808493:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575846.808497:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575846.808500:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575846.808505:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575846/real 1788575846]  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575846.808515:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575846.808517:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575846.808519:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575846.808521:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575846.808523:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575846.808526:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575846.808529:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575846.808538:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808545:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575846.808547:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.808549:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575846.808551:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575846.808556:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808563:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575846.808567:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808574:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575846.808577:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.808579:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575846.808580:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.808583:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575846.808585:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575846.808590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575846.808595:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575846.808598:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575846.808603:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575846.808611:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393240064:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575846.808616:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575846.808621:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808628:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575846.808630:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575846.808632:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575846.808642:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575846.808645:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575846.808648:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240192:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575846.808652:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575846.808655:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575846.808656:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575846.808658:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575846.808662:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575846.808664:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575846.808667:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575846.808670:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575846.808673:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575846.808677:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393240192, portal 10
00000100:00000001:2.0:1788575846.808679:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575846.808680:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:2.0:1788575846.808696:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808704:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575846.808706:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575846.808709:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575846.808711:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393240192, offset 0
00000400:00000200:2.0:1788575846.808715:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575846.808719:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575846.808723:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575846.808725:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575846.808728:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575846.808730:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575846.808732:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575846.808736:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331132:331132:256:4294967295] 192.168.202.110@tcp LPNI seq info [331089:331089:8:4294967295]
00000400:00000200:2.0:1788575846.808740:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575846.808746:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575846.808750:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575846.808753:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575846.808756:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575846.808759:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575846.808762:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575846.808765:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575846.808768:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575846.808770:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575846.808773:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575846.808776:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575846.808778:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575846.808780:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575846.808782:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575846.808785:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808793:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575846.808796:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808802:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575846.808804:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575846.808807:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575846.808812:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575846.808816:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575846.808817:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307450409
00000100:00000001:2.0:1788575846.808821:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307451324
00000100:00000001:2.0:1788575846.808823:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307450409
00000100:00000001:2.0:1788575846.808825:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307451324
00000100:00000001:2.0:1788575846.808827:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575846.808829:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575846.808831:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575846.808835:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575846/real 1788575846]  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575846.808842:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575846.808843:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575846.808845:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575846.808847:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575846.808848:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575846.808850:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575846.808854:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575846.808862:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808868:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575846.808870:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.808871:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575846.808872:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575846.808874:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808879:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575846.808881:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808887:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575846.808888:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.808889:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575846.808890:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.808891:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575846.808893:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575846.808894:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575846.808896:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575846.808897:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575846.808900:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575846.808904:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240192:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575846.808908:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575846.808913:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808919:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575846.808920:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575846.808923:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575846.808925:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575846.808926:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575846.808929:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240320:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575846.808933:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575846.808935:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575846.808937:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575846.808938:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575846.808942:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575846.808944:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575846.808948:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575846.808951:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575846.808955:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575846.808958:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393240320, portal 4
00000100:00000001:2.0:1788575846.808960:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575846.808961:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000040:2.0:1788575846.808967:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.808973:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575846.808975:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575846.808977:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575846.808980:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393240320, offset 0
00000400:00000200:2.0:1788575846.808984:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575846.808988:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575846.808992:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575846.808995:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575846.808998:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575846.808999:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575846.809001:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575846.809003:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331133:331133:256:4294967295] 192.168.202.110@tcp LPNI seq info [331090:331090:8:4294967295]
00000400:00000200:2.0:1788575846.809005:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575846.809010:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575846.809013:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575846.809015:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575846.809018:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575846.809019:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575846.809022:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575846.809024:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575846.809026:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575846.809028:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575846.809030:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575846.809032:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575846.809034:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575846.809035:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575846.809037:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575846.809040:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.809044:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575846.809046:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.809050:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575846.809051:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575846.809053:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575846.809055:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575846.809058:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575846.809060:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307450555
00000100:00000001:2.0:1788575846.809063:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307451324
00000100:00000001:2.0:1788575846.809065:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307450555
00000100:00000001:2.0:1788575846.809067:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307451324
00000100:00000001:2.0:1788575846.809069:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575846.809071:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575846.809072:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575846.809076:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575846/real 1788575846]  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575846.809083:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575846.809085:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575846.809086:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575846.809089:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575846.809090:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575846.809092:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575846.809095:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575846.809104:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.809113:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575846.809114:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.809115:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575846.809116:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575846.809118:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.809125:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575846.809128:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.809135:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575846.809137:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.809139:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575846.809140:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.809142:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575846.809143:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575846.809146:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575846.809149:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575846.809152:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575846.809155:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575846.809163:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240320:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575846.809168:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.809170:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575846.809173:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393240064/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.809179:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575846.809183:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575846.809187:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575846.809190:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575846.809193:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575846.809195:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575846.809197:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575846.809200:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:2.0:1788575846.809205:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575846.809207:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575846.809209:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:2.0:1788575846.809214:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575846.809215:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.809217:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575846.809221:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393240192/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.809227:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575846.809228:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575846.809229:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575846.809232:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575846.809233:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575846.809234:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575846.809236:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575846.809238:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:2.0:1788575846.809240:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575846.809242:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575846.809243:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:2.0:1788575846.809245:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575846.809247:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.809248:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575846.809251:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393240320/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575846.809258:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575846.809259:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575846.809260:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575846.809261:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575846.809262:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575846.809263:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575846.809265:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575846.809266:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:2.0:1788575846.809267:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575846.809268:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575846.809269:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:2.0:1788575846.809270:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575846.809271:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.809273:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575846.809280:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575846.809282:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575846.809284:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575846.809285:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575847.189590:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575847.189598:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575847.189603:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575847.189604:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575847.189608:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575847.189609:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575847.189716:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575847.189719:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575847.189722:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575847.317527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575847.317535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575847.317537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575847.317543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575847.317546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575847.317550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575847.317553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575847.317555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575847.317557:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575847.765519:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307451324, at 4307451328
00000100:00000010:0.0:1788575847.765539:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000100:00000040:1.0:1788575847.766077:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307451324, at 4307451328
00000100:00000010:1.0:1788575847.766091:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00010000:00000001:3.0:1788575848.213511:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575848.213517:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575848.213522:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575848.213524:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575848.213527:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575848.213528:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575848.213589:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575848.213592:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575848.213595:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575848.343712:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575848.343719:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575848.343721:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575848.343726:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575848.343730:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575848.343733:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575848.343735:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575848.343738:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575848.343741:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575849.237729:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575849.237737:0:216421:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575849.237739:0:216421:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575849.237742:0:216421:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575849.237747:0:216421:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575849.237749:0:216421:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575849.237751:0:216421:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575849.237754:0:216421:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575849.237757:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575849.237759:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575849.237769:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575849.237771:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575849.237833:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575849.237838:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575849.237841:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575849.365520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575849.365532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575849.365534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575849.365540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575849.365543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575849.365547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575849.365550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575849.365552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575849.365555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575850.261480:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575850.261486:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575850.261491:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575850.261492:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575850.261495:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575850.261497:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575850.261617:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575850.261620:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575850.261623:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575850.389519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575850.389526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575850.389528:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575850.389533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575850.389536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575850.389540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575850.389542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575850.389545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575850.389548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575851.285490:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575851.285494:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575851.285497:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575851.285498:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575851.285500:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575851.285501:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575851.285538:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575851.285551:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575851.285555:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575851.413476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575851.413481:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575851.413483:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575851.413487:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575851.413489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575851.413492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.413494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575851.413496:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.413498:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575851.925511:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575851.925521:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575851.925523:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575851.925526:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575851.925555:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575851.925560:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12783
00000400:00000200:3.0:1788575851.925566:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575851.925571:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575851.925573:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575851.925576:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925579:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575851.925581:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925583:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575851.925587:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575851.925589:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575851.925591:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575851.925595:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575851.925597:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575851.925599:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575851.925603:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925605:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575851.925606:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925608:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575851.925612:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575851.925615:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925618:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575851.925620:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575851.925623:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575851.925625:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575851.925627:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575851.925629:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575851.925636:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1180.
00000020:00000040:3.0:1788575851.925639:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575851.925641:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575851.925642:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575851.925645:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575851.925647:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575851.925650:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575851.925656:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575851.925661:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575851.925666:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575851.925673:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575851.925677:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575851.925680:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca1dca9a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575851.925695:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575851.925697:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575851.925699:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575851.925700:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575851.925701:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575851.925703:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575851.925705:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12783
00000400:00000200:3.0:1788575851.925707:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575851.925710:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575851.925711:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575851.925713:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925714:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575851.925716:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925717:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575851.925719:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575851.925720:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575851.925721:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575851.925722:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575851.925724:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575851.925725:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575851.925726:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925728:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575851.925729:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575851.925731:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575851.925731:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575851.925734:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575851.925737:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1788575851.925737:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925741:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575851.925742:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1788575851.925742:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575851.925748:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575851.925750:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575851.925752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575851.925752:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575851.925754:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575851.925759:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575851.925763:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575851.925767:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000020:00000040:3.0:1788575851.925770:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575851.925772:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1788575851.925773:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575851.925774:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575851.925775:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575851.925778:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575851.925778:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575851.925779:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575851.925783:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575851.925783:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575851.925785:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575851.925789:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575851.925790:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575851.925798:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393240448:192.168.202.110@tcp:250:kworker.0
00000100:00100000:3.0:1788575851.925798:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575851.925808:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575851.925812:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000001:2.0:1788575851.925813:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1788575851.925815:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575851.925818:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575851.925822:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575851.925823:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575851.925830:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
00000100:00000001:3.0:1788575851.925833:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575851.925834:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575851.925839:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575851.925844:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575851.925847:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:2.0:1788575851.925849:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00000001:3.0:1788575851.925850:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575851.925854:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:2.0:1788575851.925855:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000001:3.0:1788575851.925857:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000200:2.0:1788575851.925861:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393240448, portal 25
00000100:00080000:3.0:1788575851.925862:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12783
00000100:00000001:2.0:1788575851.925864:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575851.925866:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000400:00000200:3.0:1788575851.925870:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575851.925873:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575851.925878:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1788575851.925882:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1788575851.925884:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575851.925886:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575851.925890:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000040:3.0:1788575851.925892:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575851.925893:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393240448, offset 0
00000100:00000001:3.0:1788575851.925898:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575851.925902:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575851.925904:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925908:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575851.925912:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1788575851.925912:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575851.925915:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575851.925916:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575851.925918:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:2.0:1788575851.925920:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:3.0:1788575851.925921:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575851.925926:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575851.925928:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1788575851.925934:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575851.925935:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575851.925939:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575851.925940:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575851.925943:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:3.0:1788575851.925946:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575851.925951:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331134:331134:256:4294967295] 192.168.202.110@tcp LPNI seq info [331091:331091:8:4294967295]
00000100:00000001:3.0:1788575851.925952:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575851.925957:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575851.925962:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575851.925965:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575851.925969:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575851.925971:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575851.925976:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575851.925977:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1788575851.925981:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575851.925985:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:2.0:1788575851.925988:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00100000:3.0:1788575851.925988:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000010:2.0:1788575851.925994:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000008:00000001:3.0:1788575851.925994:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575851.925998:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575851.926002:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:3.0:1788575851.926003:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000020:00000040:3.0:1788575851.926007:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.0:1788575851.926008:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
02000000:00000001:3.0:1788575851.926010:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575851.926012:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575851.926015:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:2.1:1788575851.926016:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:3.0:1788575851.926017:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:2.1:1788575851.926019:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
02000000:00000001:3.0:1788575851.926020:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575851.926026:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000800:00000100:2.0:1788575851.926028:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575851.926030:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575851.926032:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000100:00100000:3.0:1788575851.926036:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575851.926041:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1788575851.926042:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575851.926045:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575851.926050:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575851.926051:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1788575851.926059:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575851.926060:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000100:00000040:3.0:1788575851.926062:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:2.0:1788575851.926064:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000040:3.0:1788575851.926066:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575851.926072:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:3.0:1788575851.926072:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12793)
00000100:00000200:2.0:1788575851.926077:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926084:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575851.926086:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926090:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575851.926091:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575851.926096:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575851.926099:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575851.926102:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307456441
00000100:00000010:2.0:1788575851.926106:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ce40.
00000100:00000001:2.0:1788575851.926112:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575851.926116:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307456441
00000100:00000010:2.0:1788575851.926117:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c240.
00000100:00000001:2.0:1788575851.926119:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575851.926122:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575851.926125:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575851.926127:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575851.926131:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575851/real 1788575851]  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575851.926145:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575851.926147:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575851.926150:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575851.926152:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575851.926153:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575851.926154:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575851.926157:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575851.926162:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926166:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575851.926167:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926168:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575851.926170:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575851.926172:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926179:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575851.926183:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926189:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575851.926192:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926194:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575851.926195:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926197:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575851.926199:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575851.926202:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575851.926205:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575851.926208:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575851.926212:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575851.926219:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393240448:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575851.926223:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926225:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575851.926228:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393240448/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926234:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575851.926236:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575851.926239:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575851.926242:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575851.926244:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575851.926245:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575851.926247:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575851.926249:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575851.926253:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575851.926255:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575851.926256:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:2.0:1788575851.926259:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575851.926260:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926262:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926269:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575851.926271:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575851.926273:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575851.926274:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575851.926276:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575851.926279:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926286:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575851.926287:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575851.926289:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575851.926290:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575851.926293:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575851.926296:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240576:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575851.926299:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575851.926301:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575851.926302:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575851.926304:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575851.926307:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575851.926309:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575851.926313:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575851.926315:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575851.926319:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575851.926322:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393240576, portal 10
00000100:00000001:2.0:1788575851.926323:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575851.926325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:2.0:1788575851.926330:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926336:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575851.926339:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575851.926341:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575851.926343:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393240576, offset 0
00000400:00000200:2.0:1788575851.926357:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575851.926361:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575851.926365:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575851.926367:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575851.926370:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575851.926372:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575851.926374:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575851.926377:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331135:331135:256:4294967295] 192.168.202.110@tcp LPNI seq info [331092:331092:8:4294967295]
00000400:00000200:2.0:1788575851.926380:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575851.926385:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575851.926387:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575851.926390:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575851.926425:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575851.926428:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575851.926433:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575851.926435:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575851.926439:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575851.926445:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575851.926448:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575851.926452:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575851.926455:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575851.926457:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575851.926459:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575851.926463:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926470:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575851.926473:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926479:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575851.926480:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575851.926483:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575851.926487:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575851.926490:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575851.926492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307455527
00000100:00000001:2.0:1788575851.926495:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307456441
00000100:00000001:2.0:1788575851.926497:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307455527
00000100:00000001:2.0:1788575851.926499:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307456441
00000100:00000001:2.0:1788575851.926501:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575851.926503:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575851.926504:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575851.926508:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575851/real 1788575851]  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575851.926515:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575851.926516:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575851.926518:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575851.926520:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575851.926522:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575851.926524:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575851.926526:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575851.926548:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926555:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575851.926556:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926557:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575851.926559:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575851.926562:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926568:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575851.926571:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926577:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575851.926580:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926581:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575851.926582:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926584:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575851.926586:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575851.926589:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575851.926592:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575851.926594:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575851.926597:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575851.926605:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240576:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575851.926609:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575851.926612:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926618:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575851.926620:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575851.926621:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575851.926623:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575851.926625:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575851.926628:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575851.926631:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575851.926633:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575851.926634:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575851.926636:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575851.926639:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575851.926641:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575851.926645:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575851.926648:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575851.926651:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575851.926653:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393240704, portal 4
00000100:00000001:2.0:1788575851.926655:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575851.926656:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:2.0:1788575851.926662:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926668:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575851.926670:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575851.926672:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575851.926674:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393240704, offset 0
00000400:00000200:2.0:1788575851.926678:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575851.926681:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575851.926685:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575851.926688:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575851.926691:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575851.926693:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575851.926695:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575851.926698:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331136:331136:256:4294967295] 192.168.202.110@tcp LPNI seq info [331093:331093:8:4294967295]
00000400:00000200:2.0:1788575851.926702:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575851.926709:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575851.926713:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575851.926716:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575851.926719:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575851.926722:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575851.926726:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575851.926729:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575851.926732:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575851.926734:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575851.926737:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575851.926740:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575851.926743:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575851.926744:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575851.926747:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575851.926751:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926758:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575851.926761:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926767:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575851.926769:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575851.926772:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575851.926774:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575851.926777:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575851.926779:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307455673
00000100:00000001:2.0:1788575851.926782:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307456441
00000100:00000001:2.0:1788575851.926784:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307455673
00000100:00000001:2.0:1788575851.926786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307456441
00000100:00000001:2.0:1788575851.926788:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575851.926790:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575851.926792:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575851.926796:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575851/real 1788575851]  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575851.926803:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575851.926804:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575851.926806:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575851.926808:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575851.926810:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575851.926812:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575851.926816:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575851.926824:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926828:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575851.926829:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926831:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575851.926832:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575851.926836:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926843:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575851.926845:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926849:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575851.926851:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926852:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575851.926853:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926854:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575851.926855:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575851.926857:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575851.926859:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575851.926860:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575851.926863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575851.926867:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575851.926869:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926870:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575851.926872:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393240576/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926875:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575851.926876:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575851.926878:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575851.926879:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575851.926880:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575851.926881:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575851.926882:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575851.926883:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575851.926885:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575851.926886:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575851.926887:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:2.0:1788575851.926890:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575851.926891:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926893:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575851.926895:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393240704/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575851.926901:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575851.926903:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575851.926904:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575851.926907:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575851.926908:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575851.926909:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575851.926911:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575851.926912:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575851.926914:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575851.926915:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575851.926916:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:2.0:1788575851.926919:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575851.926920:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926922:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575851.926925:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575851.926926:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575851.926928:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575851.926929:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575852.309558:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575852.309564:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575852.309570:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575852.309571:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575852.309575:0:216421:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575852.309576:0:216421:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575852.309632:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575852.309635:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575852.309637:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575852.437498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575852.437505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575852.437507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575852.437513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575852.437517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575852.437535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575852.437539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575852.437541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575852.437544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575852.885533:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307456441, at 4307456448
00000100:00000040:0.0:1788575852.885546:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307456441, at 4307456448
00000100:00000010:1.0:1788575852.885551:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00000100:00000010:0.0:1788575852.885566:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00010000:00000001:3.0:1788575853.333590:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575853.333600:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575853.333607:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575853.333608:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575853.333612:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575853.333614:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575853.333686:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575853.333704:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575853.333708:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575853.461591:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575853.461602:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575853.461604:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575853.461611:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575853.461617:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575853.461622:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575853.461625:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575853.461629:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575853.461632:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575854.357544:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575854.357551:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575854.357557:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575854.357558:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575854.357561:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575854.357562:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575854.357638:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575854.357645:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575854.357651:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575854.485564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575854.485576:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575854.485579:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575854.485585:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575854.485589:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575854.485594:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575854.485597:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575854.485600:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575854.485604:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575855.381518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575855.381523:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575855.381526:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575855.381527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575855.381529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575855.381530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575855.381600:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575855.381604:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575855.381607:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575855.509487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575855.509495:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575855.509497:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575855.509501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575855.509505:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575855.509508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575855.509511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575855.509513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575855.509516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575856.405617:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575856.405623:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575856.405629:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575856.405631:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575856.405634:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575856.405637:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575856.405639:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575856.405641:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575856.405643:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575856.405645:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575856.405649:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575856.405650:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575856.405651:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575856.405652:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575856.405654:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575856.405655:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575856.405656:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575856.405658:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575856.405713:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575856.405717:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575856.405720:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575856.533525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575856.533532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575856.533535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575856.533540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575856.533545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575856.533551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575856.533554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575856.533557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575856.533560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575857.045579:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575857.045595:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575857.045598:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575857.045601:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575857.045606:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575857.045613:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12788
00000400:00000200:2.0:1788575857.045629:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575857.045635:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575857.045638:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575857.045641:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045645:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575857.045652:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045654:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575857.045659:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575857.045661:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575857.045664:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575857.045667:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575857.045669:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575857.045672:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575857.045675:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045677:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575857.045679:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045681:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575857.045684:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575857.045688:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045691:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575857.045693:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575857.045696:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575857.045699:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575857.045701:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575857.045703:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575857.045712:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00e00.
00000020:00000040:2.0:1788575857.045721:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575857.045725:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575857.045727:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575857.045730:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575857.045732:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575857.045736:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575857.045743:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788575857.045755:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575857.045765:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575857.045775:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575857.045785:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575857.045790:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5f665f0] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575857.045816:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575857.045819:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575857.045823:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575857.045824:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575857.045826:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575857.045838:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575857.045846:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12788
00000400:00000200:2.0:1788575857.045850:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575857.045853:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575857.045855:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575857.045858:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045861:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575857.045863:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045865:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575857.045867:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575857.045868:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575857.045869:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575857.045871:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575857.045879:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575857.045881:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575857.045883:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045885:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575857.045887:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045889:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575857.045892:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575857.045894:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575857.045896:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575857.045898:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575857.045903:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575857.045912:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575857.045914:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575857.045917:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575857.045920:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575857.045922:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575857.045926:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01c00.
00000020:00000040:2.0:1788575857.045928:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575857.045931:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575857.045932:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575857.045933:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575857.045940:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575857.045942:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575857.045946:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788575857.045949:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575857.045954:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575857.045971:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575857.045974:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575857.045977:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000de3acaa1] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575857.045985:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575857.045988:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575857.045991:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575857.045993:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575857.045995:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575857.046003:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575857.046005:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12788
00000400:00000200:2.0:1788575857.046009:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575857.046012:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575857.046014:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575857.046016:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.046019:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575857.046020:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.046022:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575857.046024:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575857.046026:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575857.046027:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575857.046034:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575857.046036:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575857.046038:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575857.046040:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.046042:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575857.046044:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575857.046046:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575857.046048:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575857.046050:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575857.046053:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575857.046055:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575857.046057:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575857.046065:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575857.046067:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575857.046069:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575857.046071:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575857.046074:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575857.046077:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00a80.
00000020:00000040:2.0:1788575857.046079:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575857.046082:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575857.046083:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575857.046085:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575857.046086:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575857.046088:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575857.046097:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:2.0:1788575857.046100:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575857.046104:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575857.046112:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575857.046113:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575857.046117:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000389a8694] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575857.046131:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575857.046134:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575857.046137:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575857.046142:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12798)
00000100:00000001:3.0:1788575857.046439:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575857.046443:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575857.046445:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575857.046452:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.046460:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575857.046462:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575857.046465:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575857.046467:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575857.046469:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575857.046473:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393240832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575857.046477:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575857.046481:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575857.046483:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575857.046485:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575857.046489:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575857.046491:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575857.046496:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575857.046556:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575857.046564:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575857.046571:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393240832, portal 25
00000100:00000001:3.0:1788575857.046573:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575857.046575:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:3.0:1788575857.046581:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.046588:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575857.046591:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575857.046593:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575857.046597:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393240832, offset 0
00000400:00000200:3.0:1788575857.046603:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575857.046611:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575857.046618:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575857.046623:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575857.046628:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575857.046631:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575857.046634:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575857.046638:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331137:331137:256:4294967295] 192.168.202.110@tcp LPNI seq info [331094:331094:8:4294967295]
00000400:00000200:3.0:1788575857.046642:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575857.046651:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575857.046863:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575857.046869:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575857.046876:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575857.046879:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788575857.046884:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575857.046887:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1788575857.046890:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575857.046893:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575857.046898:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575857.046902:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575857.046907:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575857.046909:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575857.046913:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575857.046927:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.046935:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575857.046938:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.046944:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575857.046946:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575857.046953:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575857.046956:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575857.046959:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307461562
00000100:00000010:3.0:1788575857.046964:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575857.046969:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575857.046974:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307461562
00000100:00000010:3.0:1788575857.046977:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575857.046979:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575857.046982:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575857.046985:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575857.046987:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575857.046991:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575857/real 1788575857]  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575857.046999:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575857.047001:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575857.047003:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575857.047005:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575857.047007:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575857.047009:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575857.047012:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575857.047018:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047033:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575857.047035:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.047037:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575857.047039:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575857.047042:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047048:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575857.047052:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047058:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575857.047060:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.047062:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575857.047063:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.047065:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575857.047067:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575857.047069:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575857.047073:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575857.047075:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575857.047078:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575857.047085:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393240832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575857.047089:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575857.047092:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047103:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575857.047105:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575857.047107:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575857.047108:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575857.047110:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575857.047113:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575857.047116:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575857.047118:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575857.047119:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575857.047121:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575857.047124:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575857.047126:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575857.047129:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575857.047132:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575857.047134:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575857.047137:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393240960, portal 10
00000100:00000001:3.0:1788575857.047138:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575857.047139:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:3.0:1788575857.047145:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047151:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575857.047153:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575857.047154:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575857.047156:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393240960, offset 0
00000400:00000200:3.0:1788575857.047159:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575857.047163:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575857.047166:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575857.047585:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575857.047589:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575857.047591:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575857.047593:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575857.047596:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331138:331138:256:4294967295] 192.168.202.110@tcp LPNI seq info [331095:331095:8:4294967295]
00000400:00000200:3.0:1788575857.047601:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575857.047607:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575857.047612:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575857.047616:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575857.047620:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575857.047623:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788575857.047627:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575857.047629:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1788575857.047633:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575857.047634:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575857.047638:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575857.047641:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575857.047644:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575857.047646:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575857.047649:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575857.047653:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047661:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575857.047663:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047670:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575857.047671:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575857.047674:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575857.047678:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575857.047681:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575857.047683:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307460648
00000100:00000001:3.0:1788575857.047686:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307461562
00000100:00000001:3.0:1788575857.047688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307460648
00000100:00000001:3.0:1788575857.047690:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307461562
00000100:00000001:3.0:1788575857.047692:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575857.047694:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575857.047696:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575857.047699:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575857/real 1788575857]  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575857.047706:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575857.047708:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575857.047709:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575857.047712:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575857.047713:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575857.047715:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575857.047718:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575857.047726:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047737:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575857.047739:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.047740:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575857.047742:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575857.047745:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047752:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575857.047755:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047761:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575857.047764:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.047765:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575857.047766:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.047768:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575857.047770:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575857.047772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575857.047775:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575857.047777:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575857.047781:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575857.047788:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393240960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575857.047791:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575857.047794:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047801:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575857.047802:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575857.047805:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575857.047807:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575857.047808:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575857.047811:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241088:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575857.047814:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575857.047816:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575857.047817:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575857.047819:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575857.047822:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575857.047824:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575857.047827:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575857.047830:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575857.047833:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575857.047836:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393241088, portal 4
00000100:00000001:3.0:1788575857.047838:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575857.047839:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:3.0:1788575857.047845:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047850:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575857.047852:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575857.047854:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575857.047856:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393241088, offset 0
00000400:00000200:3.0:1788575857.047859:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575857.047863:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575857.047866:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575857.047868:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575857.047871:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575857.047873:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575857.047875:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575857.047878:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331139:331139:256:4294967295] 192.168.202.110@tcp LPNI seq info [331096:331096:8:4294967295]
00000400:00000200:3.0:1788575857.047881:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575857.047887:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575857.047891:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575857.047895:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575857.047898:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575857.047901:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1788575857.047905:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575857.047907:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1788575857.047910:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575857.047912:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575857.047915:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575857.047930:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575857.047933:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575857.047934:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575857.047936:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575857.047940:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575882 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047947:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575857.047949:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575882 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.047955:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575857.047957:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575857.047960:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575857.047963:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575857.047966:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575857.047968:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307460794
00000100:00000001:3.0:1788575857.047970:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307461562
00000100:00000001:3.0:1788575857.047972:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307460794
00000100:00000001:3.0:1788575857.047974:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307461562
00000100:00000001:3.0:1788575857.047976:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575857.047977:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575857.047979:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575857.047981:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575857/real 1788575857]  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575857.047987:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575857.047988:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575857.047989:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575857.047990:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575857.047991:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575857.047993:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575857.047994:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575857.047999:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.048002:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575857.048003:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.048004:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575857.048005:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575857.048007:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.048010:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575857.048012:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.048017:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575857.048018:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.048019:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575857.048020:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.048021:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575857.048022:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575857.048023:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575857.048031:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575857.048034:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575857.048038:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575857.048045:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241088:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575857.048049:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.048051:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575857.048054:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393240832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.048061:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575857.048063:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575857.048065:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788575857.048069:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575857.048071:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575857.048072:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575857.048075:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575857.048076:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575857.048079:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575857.048080:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575857.048081:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:3.0:1788575857.048083:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575857.048084:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.048085:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575857.048087:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393240960/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.048092:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575857.048092:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575857.048093:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8800.
02000000:00000001:3.0:1788575857.048095:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575857.048096:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575857.048096:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575857.048098:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575857.048098:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575857.048106:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575857.048107:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575857.048109:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:3.0:1788575857.048111:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575857.048112:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.048113:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575857.048116:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393241088/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575882 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575857.048122:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575857.048123:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575857.048125:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb400.
02000000:00000001:3.0:1788575857.048127:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575857.048128:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575857.048129:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575857.048131:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575857.048133:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575857.048134:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575857.048136:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575857.048137:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:3.0:1788575857.048139:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575857.048140:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.048142:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575857.048149:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575857.048150:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575857.048151:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575857.048153:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575857.429516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575857.429521:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575857.429526:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575857.429527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575857.429530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575857.429531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575857.429580:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575857.429583:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575857.429585:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575857.557888:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575857.557895:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575857.557907:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575857.557911:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575857.557915:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575857.557918:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575857.557920:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575857.557923:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575857.557926:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575858.005494:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307461562, at 4307461568
00000100:00000040:1.0:1788575858.005494:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307461562, at 4307461568
00000100:00000010:1.0:1788575858.005511:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:0.0:1788575858.005513:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575858.453527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575858.453533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575858.453538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575858.453540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575858.453543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575858.453544:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575858.453717:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575858.453721:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575858.453724:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575858.581514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575858.581521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575858.581523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575858.581529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575858.581532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575858.581535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575858.581538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575858.581540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575858.581543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575859.477625:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575859.477633:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575859.477636:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575859.477639:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575859.477643:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575859.477646:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575859.477648:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575859.477650:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575859.477653:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575859.477655:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575859.477658:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575859.477659:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575859.477717:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575859.477719:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575859.477721:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575859.605490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575859.605497:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575859.605499:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575859.605504:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575859.605508:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575859.605511:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575859.605514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575859.605518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575859.605520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575860.501543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575860.501549:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575860.501553:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575860.501555:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575860.501558:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575860.501559:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575860.501623:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575860.501627:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575860.501631:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575860.629523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575860.629530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575860.629533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575860.629538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575860.629542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575860.629547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575860.629550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575860.629552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575860.629555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575861.525504:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575861.525509:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575861.525514:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575861.525515:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575861.525518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575861.525519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575861.526389:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575861.526436:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575861.526440:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575861.653513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575861.653522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575861.653525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575861.653530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575861.653534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575861.653537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575861.653540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575861.653542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575861.653545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575862.165806:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575862.165817:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575862.165820:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575862.165823:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575862.165827:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575862.165834:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12793
00000400:00000200:3.0:1788575862.165843:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575862.165849:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575862.165852:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575862.165855:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575862.165858:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575862.165860:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575862.165863:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575862.165867:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575862.165869:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575862.165872:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575862.165875:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575862.165877:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575862.165879:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575862.165883:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575862.165885:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575862.165887:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575862.165888:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575862.165891:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575862.165895:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575862.165897:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575862.165900:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575862.165903:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575862.165905:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575862.165908:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575862.165910:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575862.165918:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788575862.165922:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575862.165936:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575862.165938:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575862.165940:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575862.165943:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575862.165947:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575862.165955:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575862.165962:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575862.165972:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575862.165982:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575862.165986:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575862.165992:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575862.166013:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575862.166016:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575862.166020:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575862.166021:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575862.166023:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575862.166026:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575862.166029:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12793
00000400:00000200:3.0:1788575862.166033:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575862.166037:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575862.166039:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575862.166041:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575862.166044:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575862.166045:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575862.166047:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575862.166050:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575862.166051:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575862.166055:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575862.166056:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788575862.166058:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575862.166061:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575862.166064:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575862.166065:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575862.166069:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575862.166074:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575862.166080:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1788575862.166082:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575862.166083:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575862.166088:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575862.166088:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575862.166092:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1788575862.166093:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575862.166094:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575862.166098:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00100000:2.0:1788575862.166100:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393241216:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1788575862.166103:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575862.166104:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575862.166109:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:3.0:1788575862.166109:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575862.166112:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575862.166115:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575862.166115:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575862.166119:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575862.166125:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575862.166127:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575862.166129:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575862.166133:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1788575862.166136:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
00000008:00000001:3.0:1788575862.166137:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575862.166142:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575862.166142:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575862.166147:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000400:00000200:2.0:1788575862.166152:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1788575862.166152:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575862.166158:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1788575862.166160:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
02000000:00000001:3.0:1788575862.166160:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575862.166166:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575862.166168:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575862.166170:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575862.166171:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
02000000:00000010:3.0:1788575862.166176:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000200:2.0:1788575862.166178:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393241216, portal 25
00000100:00000001:3.0:1788575862.166181:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.166183:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575862.166187:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00100000:3.0:1788575862.166189:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788575862.166196:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575862.166201:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575862.166205:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.166212:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575862.166213:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575862.166218:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575862.166222:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575862.166228:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393241216, offset 0
00000100:00000001:3.0:1788575862.166230:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575862.166234:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575862.166239:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575862.166240:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575862.166242:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575862.166245:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575862.166254:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575862.166260:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575862.166263:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:3.0:1788575862.166267:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12793
00000400:00000200:2.0:1788575862.166272:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575862.166278:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575862.166282:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575862.166284:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575862.166288:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331140:331140:256:4294967295] 192.168.202.110@tcp LPNI seq info [331097:331097:8:4294967295]
00000400:00000200:2.0:1788575862.166299:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575862.166305:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575862.166311:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575862.166314:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1788575862.166316:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575862.166333:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.0:1788575862.166338:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575862.166338:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575862.166344:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575862.166345:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000100:00000001:3.0:1788575862.166347:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575862.166351:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575862.166353:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:2.0:1788575862.166356:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788575862.166356:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575862.166360:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:2.0:1788575862.166362:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:3.0:1788575862.166363:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575862.166368:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.1:1788575862.166373:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575862.166375:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575862.166378:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:3.0:1788575862.166379:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575862.166383:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575862.166387:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000100:2.0:1788575862.166388:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575862.166411:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575862.166415:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575862.166418:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575862.166420:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575862.166423:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575862.166425:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575862.166427:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575862.166430:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575862.166432:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575862.166434:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575862.166437:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000020:00000040:3.0:1788575862.166440:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575862.166443:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575862.166444:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575862.166445:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575862.166447:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575862.166448:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575862.166453:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575862.166457:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575862.166462:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575862.166470:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575862.166472:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575862.166475:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575862.166483:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575862.166486:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575862.166489:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575862.166494:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12803)
00000800:00000010:2.0:1788575862.166530:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575862.166535:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575862.166539:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575862.166544:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575862.166547:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575862.166552:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575862.166556:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166566:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575862.166569:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166577:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.166578:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575862.166584:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575862.166588:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575862.166591:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307466682
00000100:00000010:2.0:1788575862.166596:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c660.
00000100:00000001:2.0:1788575862.166601:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575862.166606:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307466682
00000100:00000010:2.0:1788575862.166609:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cae0.
00000100:00000001:2.0:1788575862.166612:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575862.166615:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575862.166617:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.166619:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575862.166624:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575862/real 1788575862]  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575862.166633:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575862.166634:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575862.166637:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575862.166640:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575862.166642:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575862.166644:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575862.166648:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575862.166656:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166667:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575862.166669:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.166671:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575862.166673:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575862.166677:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166683:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575862.166687:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166693:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575862.166695:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.166697:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575862.166698:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.166700:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575862.166702:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575862.166705:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575862.166708:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575862.166710:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575862.166714:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575862.166722:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393241216:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575862.166727:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.166729:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575862.166732:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393241216/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166738:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575862.166741:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575862.166744:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575862.166747:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575862.166748:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575862.166750:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575862.166752:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575862.166754:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575862.166757:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575862.166759:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575862.166760:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788575862.166763:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575862.166764:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.166766:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.166772:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575862.166774:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.166776:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575862.166777:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575862.166779:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575862.166782:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166789:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575862.166791:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575862.166793:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575862.166794:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575862.166797:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575862.166799:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241344:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575862.166802:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575862.166804:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575862.166806:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575862.166807:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575862.166810:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575862.166811:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575862.166815:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575862.166818:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575862.166821:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575862.166824:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393241344, portal 10
00000100:00000001:2.0:1788575862.166826:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575862.166827:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:2.0:1788575862.166833:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166840:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575862.166842:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575862.166844:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575862.166847:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393241344, offset 0
00000400:00000200:2.0:1788575862.166851:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575862.166855:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575862.166858:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575862.166861:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575862.166864:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575862.166866:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575862.166869:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575862.166871:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331141:331141:256:4294967295] 192.168.202.110@tcp LPNI seq info [331098:331098:8:4294967295]
00000400:00000200:2.0:1788575862.166876:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575862.166882:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575862.166887:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575862.166891:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575862.166894:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575862.166911:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788575862.166915:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575862.166918:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1788575862.166921:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575862.166922:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575862.166925:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575862.166928:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575862.166932:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575862.166933:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575862.166936:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575862.166939:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166952:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575862.166955:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.166960:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.166962:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575862.166965:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575862.166968:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575862.166972:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575862.166974:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307465767
00000100:00000001:2.0:1788575862.166976:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307466682
00000100:00000001:2.0:1788575862.166978:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307465767
00000100:00000001:2.0:1788575862.166979:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307466682
00000100:00000001:2.0:1788575862.166981:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575862.166984:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.166985:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575862.166989:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575862/real 1788575862]  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575862.166996:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575862.166997:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575862.166999:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575862.167101:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575862.167103:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575862.167106:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575862.167110:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575862.167118:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167179:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575862.167182:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167184:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575862.167185:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575862.167190:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167196:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575862.167200:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167207:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575862.167209:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167210:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575862.167211:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167213:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575862.167215:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575862.167218:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575862.167221:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575862.167224:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575862.167227:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575862.167235:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241344:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575862.167239:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575862.167241:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167248:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575862.167249:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575862.167251:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575862.167253:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575862.167255:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575862.167258:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241472:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575862.167261:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575862.167263:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575862.167265:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575862.167266:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575862.167270:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575862.167271:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575862.167275:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575862.167277:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575862.167281:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575862.167284:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393241472, portal 4
00000100:00000001:2.0:1788575862.167286:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575862.167288:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:2.0:1788575862.167293:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167299:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575862.167302:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575862.167304:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575862.167306:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393241472, offset 0
00000400:00000200:2.0:1788575862.167310:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575862.167314:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575862.167317:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575862.167332:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575862.167335:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575862.167337:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575862.167339:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575862.167342:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331142:331142:256:4294967295] 192.168.202.110@tcp LPNI seq info [331099:331099:8:4294967295]
00000400:00000200:2.0:1788575862.167346:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575862.167353:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575862.167357:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575862.167361:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575862.167365:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575862.167367:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788575862.167371:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575862.167374:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1788575862.167379:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575862.167381:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575862.167384:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575862.167387:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575862.167408:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575862.167410:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575862.167413:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575862.167416:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575887 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167423:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575862.167426:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575887 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167433:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.167434:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575862.167438:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575862.167440:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575862.167443:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575862.167445:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307465914
00000100:00000001:2.0:1788575862.167448:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307466682
00000100:00000001:2.0:1788575862.167450:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307465914
00000100:00000001:2.0:1788575862.167452:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307466682
00000100:00000001:2.0:1788575862.167454:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575862.167455:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.167457:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575862.167461:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575862/real 1788575862]  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575862.167469:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575862.167471:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575862.167472:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575862.167475:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575862.167477:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575862.167479:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575862.167482:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575862.167490:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167508:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575862.167510:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167511:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575862.167513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575862.167517:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167523:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575862.167527:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167533:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575862.167535:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167537:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575862.167538:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167540:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575862.167541:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575862.167544:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575862.167547:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575862.167550:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575862.167554:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575862.167567:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241472:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575862.167570:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167572:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575862.167575:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393241344/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167580:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575862.167582:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575862.167584:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575862.167586:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575862.167588:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575862.167589:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575862.167591:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575862.167593:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575862.167595:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575862.167596:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575862.167598:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:2.0:1788575862.167600:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575862.167601:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167603:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575862.167605:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393241472/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575887 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575862.167611:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575862.167613:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575862.167614:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788575862.167616:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575862.167617:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575862.167618:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575862.167620:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575862.167621:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575862.167623:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575862.167624:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575862.167625:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788575862.167628:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575862.167629:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167630:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575862.167634:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575862.167635:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575862.167637:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575862.167638:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575862.549559:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575862.549565:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575862.549570:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575862.549571:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575862.549574:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575862.549575:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575862.549641:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575862.549645:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575862.549648:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575862.677526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575862.677536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575862.677539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575862.677544:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575862.677547:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575862.677551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575862.677554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575862.677557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575862.677559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575863.125500:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307466682, at 4307466688
00000100:00000010:0.0:1788575863.125511:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000100:00000040:1.0:1788575863.125550:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307466682, at 4307466688
00000100:00000010:1.0:1788575863.125558:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00010000:00000001:3.0:1788575863.573551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575863.573561:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575863.573568:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575863.573570:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575863.573573:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575863.573575:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575863.573659:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575863.573664:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575863.573668:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575863.701513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575863.701520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575863.701521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575863.701525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575863.701528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575863.701531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575863.701533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575863.701535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575863.701537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575864.597503:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575864.597510:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575864.597524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575864.597525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575864.597527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575864.597528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575864.597733:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575864.597736:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575864.597738:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575864.725507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575864.725513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575864.725514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575864.725519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575864.725521:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575864.725524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575864.725526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575864.725528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575864.725530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575865.237591:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1788575865.237598:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1788575865.237603:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:0.0:1788575865.237996:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575865.238006:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575865.238023:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575865.238028:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575865.238031:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575865.622148:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575865.622154:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575865.622159:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575865.622161:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575865.622165:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575865.622166:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575865.622232:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575865.622235:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575865.622238:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575865.749820:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575865.749828:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575865.749830:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575865.749836:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575865.749840:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575865.749844:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575865.749846:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575865.749849:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575865.749852:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575866.645497:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575866.645504:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575866.645511:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575866.645514:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575866.645517:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575866.645520:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575866.645523:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575866.645525:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575866.645527:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575866.645530:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575866.645534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575866.645536:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575866.645537:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575866.645539:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575866.645541:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575866.645542:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575866.645543:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575866.645545:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575866.645597:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575866.645600:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575866.645603:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575866.773483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575866.773490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575866.773492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575866.773498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575866.773501:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575866.773505:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575866.773509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575866.773511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575866.773513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575867.285508:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575867.285518:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575867.285521:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575867.285525:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575867.285530:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575867.285537:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12798
00000400:00000200:2.0:1788575867.285546:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575867.285553:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575867.285557:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575867.285560:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575867.285564:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575867.285566:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575867.285569:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575867.285572:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575867.285575:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575867.285578:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575867.285581:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575867.285584:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575867.285586:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575867.285589:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575867.285591:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575867.285593:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575867.285595:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575867.285598:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575867.285602:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575867.285605:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575867.285608:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575867.285611:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575867.285613:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575867.285616:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575867.285618:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575867.285627:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03800.
00000020:00000040:2.0:1788575867.285631:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575867.285635:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575867.285637:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575867.285639:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575867.285641:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575867.285645:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575867.285652:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788575867.285658:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575867.285667:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575867.285678:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575867.285682:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575867.285687:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014d98894] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575867.285717:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575867.285721:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575867.285724:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575867.285725:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575867.285727:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575867.285729:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575867.285732:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12798
00000400:00000200:2.0:1788575867.285735:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575867.285739:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575867.285742:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575867.285745:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575867.285748:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575867.285750:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575867.285752:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575867.285753:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575867.285759:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1788575867.285759:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1788575867.285760:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575867.285763:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788575867.285764:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575867.285768:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575867.285772:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575867.285772:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575867.285776:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575867.285784:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575867.285789:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575867.285791:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1788575867.285793:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575867.285795:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575867.285798:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:3.0:1788575867.285802:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575867.285803:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575867.285808:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575867.285810:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575867.285812:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575867.285814:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575867.285818:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575867.285819:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575867.285824:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575867.285824:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393241600:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575867.285828:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575867.285834:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1788575867.285834:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1788575867.285839:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575867.285840:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1788575867.285844:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575867.285844:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575867.285850:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:2.0:1788575867.285851:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02a00.
00000020:00000040:2.0:1788575867.285855:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000010:3.0:1788575867.285857:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575867.285859:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575867.285862:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575867.285863:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575867.285868:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575867.285870:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575867.285873:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575867.285873:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575867.285880:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
02000000:00000010:2.0:1788575867.285883:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12c00.
00000400:00000010:3.0:1788575867.285889:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575867.285890:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575867.285898:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575867.285898:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393241600, portal 25
00000100:00000001:3.0:1788575867.285903:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575867.285905:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:3.0:1788575867.285912:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575867.285931:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575867.285931:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1788575867.285936:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575867.285937:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575867.285942:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000040:2.0:1788575867.285944:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cce910e4] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575867.285947:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393241600, offset 0
00000100:00000001:2.0:1788575867.285958:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575867.285958:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575867.285964:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575867.285968:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:3.0:1788575867.285970:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575867.285971:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575867.285974:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575867.285977:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:3.0:1788575867.285980:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1788575867.285981:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12798
00000400:00000200:3.0:1788575867.285990:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575867.285995:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575867.285998:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575867.286001:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575867.286004:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331143:331143:256:4294967295] 192.168.202.110@tcp LPNI seq info [331100:331100:8:4294967295]
00000400:00000200:3.0:1788575867.286009:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575867.286015:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575867.286023:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1788575867.286025:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1788575867.286027:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575867.286034:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575867.286038:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575867.286038:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575867.286045:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:3.0:1788575867.286046:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000100:00000001:2.0:1788575867.286048:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575867.286052:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575867.286054:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575867.286057:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.0:1788575867.286058:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1788575867.286060:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:3.0:1788575867.286064:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:2.0:1788575867.286065:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575867.286070:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575867.286076:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:3.1:1788575867.286076:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575867.286080:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1788575867.286082:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:2.0:1788575867.286084:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575867.286089:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575867.286093:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000100:3.0:1788575867.286094:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788575867.286099:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000010:3.0:1788575867.286099:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000100:00000001:2.0:1788575867.286105:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575867.286107:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575867.286109:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1788575867.286111:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575867.286115:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575867.286117:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1788575867.286119:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575867.286123:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:3.0:1788575867.286126:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000008:00000001:2.0:1788575867.286127:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575867.286130:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:2.0:1788575867.286131:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575867.286137:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01f80.
00000020:00000040:2.0:1788575867.286139:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575867.286142:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575867.286144:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286146:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:2.0:1788575867.286147:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575867.286150:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575867.286154:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1788575867.286154:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788575867.286161:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000100:00000001:3.0:1788575867.286169:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575867.286172:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575867.286178:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286178:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286183:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00100000:2.0:1788575867.286189:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575867.286190:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575867.286199:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575867.286204:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575867.286204:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307471801
00000100:00000001:2.0:1788575867.286208:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575867.286213:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000040:2.0:1788575867.286227:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8c8ec7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575867.286236:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575867.286242:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575867.286246:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575867.286251:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12808)
00000100:00000001:3.0:1788575867.286252:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575867.286262:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307471801
00000100:00000010:3.0:1788575867.286267:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575867.286271:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575867.286275:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575867.286278:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286281:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575867.286287:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575867/real 1788575867]  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575867.286298:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575867.286299:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575867.286301:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575867.286304:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575867.286306:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575867.286308:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575867.286311:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575867.286320:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286328:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575867.286330:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286332:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575867.286335:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575867.286338:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286345:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575867.286350:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286355:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575867.286357:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286358:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575867.286359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286360:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575867.286362:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575867.286364:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575867.286374:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575867.286378:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575867.286382:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575867.286411:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393241600:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575867.286415:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286417:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575867.286420:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393241600/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286426:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575867.286429:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575867.286432:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575867.286435:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575867.286437:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575867.286439:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575867.286441:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575867.286443:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788575867.286446:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575867.286447:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575867.286449:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:3.0:1788575867.286451:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575867.286452:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286454:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286463:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575867.286465:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286467:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575867.286468:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575867.286469:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575867.286472:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286478:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575867.286479:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575867.286481:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575867.286482:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575867.286484:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575867.286486:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575867.286489:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575867.286491:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575867.286493:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575867.286494:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575867.286497:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575867.286498:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575867.286502:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575867.286504:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575867.286508:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575867.286510:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393241728, portal 10
00000100:00000001:3.0:1788575867.286513:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575867.286514:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:3.0:1788575867.286519:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286525:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575867.286526:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575867.286529:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575867.286531:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393241728, offset 0
00000400:00000200:3.0:1788575867.286534:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575867.286538:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575867.286542:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575867.286545:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575867.286547:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575867.286549:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575867.286551:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575867.286553:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331144:331144:256:4294967295] 192.168.202.110@tcp LPNI seq info [331101:331101:8:4294967295]
00000400:00000200:3.0:1788575867.286555:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575867.286560:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575867.286563:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575867.286565:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575867.286567:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575867.286569:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788575867.286572:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575867.286574:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1788575867.286576:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575867.286577:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575867.286579:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575867.286581:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575867.286583:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575867.286584:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575867.286586:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575867.286588:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286597:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575867.286598:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286602:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286603:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575867.286605:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575867.286608:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575867.286610:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575867.286611:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307470887
00000100:00000001:3.0:1788575867.286613:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307471801
00000100:00000001:3.0:1788575867.286614:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307470887
00000100:00000001:3.0:1788575867.286616:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307471801
00000100:00000001:3.0:1788575867.286617:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575867.286618:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286619:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575867.286621:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575867/real 1788575867]  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575867.286625:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575867.286626:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575867.286627:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575867.286629:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575867.286630:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575867.286631:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575867.286633:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575867.286637:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286640:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575867.286642:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286643:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575867.286644:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575867.286646:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286649:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575867.286652:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286655:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575867.286656:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286657:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575867.286658:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286659:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575867.286660:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575867.286662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575867.286663:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575867.286665:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575867.286667:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575867.286671:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575867.286673:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575867.286675:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286679:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575867.286680:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575867.286681:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575867.286682:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575867.286683:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575867.286685:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575867.286687:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575867.286688:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575867.286689:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575867.286690:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575867.286692:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575867.286693:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575867.286695:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575867.286696:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575867.286698:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575867.286700:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393241856, portal 4
00000100:00000001:3.0:1788575867.286701:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575867.286702:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:3.0:1788575867.286705:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286709:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575867.286710:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575867.286711:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575867.286713:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393241856, offset 0
00000400:00000200:3.0:1788575867.286715:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575867.286717:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575867.286718:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575867.286720:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575867.286722:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575867.286723:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575867.286724:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575867.286726:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331145:331145:256:4294967295] 192.168.202.110@tcp LPNI seq info [331102:331102:8:4294967295]
00000400:00000200:3.0:1788575867.286728:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575867.286732:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575867.286734:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575867.286736:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575867.286738:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575867.286740:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1788575867.286742:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575867.286744:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1788575867.286746:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575867.286747:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575867.286749:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575867.286751:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575867.286752:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575867.286753:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575867.286755:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575867.286756:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575892 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286761:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575867.286762:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575892 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286766:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286767:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575867.286769:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575867.286770:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575867.286772:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575867.286773:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307471033
00000100:00000001:3.0:1788575867.286775:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307471801
00000100:00000001:3.0:1788575867.286776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307471033
00000100:00000001:3.0:1788575867.286777:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307471801
00000100:00000001:3.0:1788575867.286778:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575867.286779:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286780:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575867.286782:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575867/real 1788575867]  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575867.286786:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575867.286787:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575867.286788:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575867.286789:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575867.286790:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575867.286791:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575867.286793:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575867.286797:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286800:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575867.286801:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575867.286803:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575867.286805:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286808:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575867.286810:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286813:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575867.286815:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286815:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575867.286816:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286817:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575867.286818:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575867.286819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575867.286821:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575867.286822:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575867.286824:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575867.286833:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393241856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575867.286835:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286836:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575867.286838:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393241728/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286841:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575867.286842:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575867.286843:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575867.286844:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575867.286845:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575867.286845:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575867.286846:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575867.286847:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:3.0:1788575867.286849:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575867.286849:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575867.286850:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:3.0:1788575867.286852:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575867.286853:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286854:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575867.286855:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393241856/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575892 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575867.286859:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575867.286859:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575867.286860:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575867.286861:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575867.286862:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575867.286862:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575867.286863:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575867.286864:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:3.0:1788575867.286865:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575867.286866:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575867.286866:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575867.286868:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575867.286869:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286870:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575867.286872:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575867.286873:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575867.286874:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575867.286875:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575867.669477:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575867.669483:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575867.669488:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575867.669489:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575867.669492:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575867.669494:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575867.669551:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575867.669555:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575867.669558:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575867.797542:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575867.797549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575867.797552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575867.797557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575867.797561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575867.797564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575867.797567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575867.797570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575867.797573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575868.245493:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307471801, at 4307471808
00000100:00000040:1.0:1788575868.245493:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307471801, at 4307471808
00000100:00000010:1.0:1788575868.245508:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00000100:00000010:0.0:1788575868.245510:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00010000:00000001:3.0:1788575868.693548:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575868.693554:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575868.693559:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575868.693560:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575868.693564:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575868.693565:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575868.693625:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575868.693628:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575868.693630:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575868.821905:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575868.821911:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575868.821913:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575868.821916:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575868.821919:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575868.821922:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575868.821924:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575868.821925:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575868.821927:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575869.717513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575869.717521:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575869.717524:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575869.717527:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575869.717532:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575869.717534:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575869.717537:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575869.717539:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575869.718204:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575869.718206:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575869.718210:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575869.718211:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575869.720266:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575869.720271:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575869.720275:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575869.845496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575869.845503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575869.845505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575869.845511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575869.845515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575869.845518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575869.845521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575869.845523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575869.845526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575870.741469:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575870.741484:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575870.741490:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575870.741491:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575870.741495:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575870.741496:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575870.741637:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575870.741641:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575870.741644:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575870.869470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575870.869477:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575870.869479:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575870.869485:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575870.869488:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575870.869492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575870.869494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575870.869497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575870.869500:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575871.765527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575871.765533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575871.765538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575871.765540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575871.765543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575871.765544:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575871.765647:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575871.765654:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575871.765660:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575871.893506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575871.893513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575871.893516:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575871.893526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575871.893530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575871.893534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575871.893537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575871.893539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575871.893546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575872.405492:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575872.405500:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575872.405501:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575872.405504:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575872.405507:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575872.405513:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12803
00000400:00000200:3.0:1788575872.405520:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575872.405526:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575872.405528:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575872.405530:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405532:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575872.405534:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405535:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575872.405538:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575872.405539:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575872.405541:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575872.405544:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575872.405545:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575872.405547:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575872.405549:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405550:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575872.405551:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405553:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575872.405555:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575872.405557:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405558:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575872.405561:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575872.405563:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575872.405565:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575872.405566:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575872.405568:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575872.405575:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000020:00000040:3.0:1788575872.405578:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575872.405581:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575872.405582:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575872.405584:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575872.405586:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575872.405589:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575872.405595:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788575872.405600:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575872.405607:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575872.405616:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575872.405620:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575872.405624:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575872.405651:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575872.405654:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575872.405657:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575872.405658:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575872.405660:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575872.405662:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575872.405665:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12803
00000400:00000200:3.0:1788575872.405668:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575872.405670:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1788575872.405671:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:3.0:1788575872.405672:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575872.405675:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575872.405676:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575872.405677:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575872.405679:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575872.405680:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405683:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575872.405685:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575872.405686:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000040:2.0:1788575872.405688:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1788575872.405688:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575872.405691:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575872.405694:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575872.405698:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575872.405703:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405706:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575872.405709:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1788575872.405709:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405712:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575872.405713:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575872.405714:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575872.405718:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575872.405720:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575872.405721:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575872.405724:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575872.405725:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575872.405728:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575872.405729:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575872.405731:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575872.405734:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:2.0:1788575872.405737:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393241984:192.168.202.110@tcp:250:kworker.0
00000008:00100000:3.0:1788575872.405738:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575872.405742:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575872.405745:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.405746:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:3.0:1788575872.405748:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
02000000:00000001:2.0:1788575872.405751:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:3.0:1788575872.405751:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575872.405754:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575872.405755:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575872.405758:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1788575872.405758:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575872.405761:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575872.405762:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:2.0:1788575872.405764:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575872.405764:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575872.405767:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575872.405768:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000400:00000200:2.0:1788575872.405772:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575872.405773:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575872.405777:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00100000:3.0:1788575872.405779:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1788575872.405784:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000001:3.0:1788575872.405786:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575872.405789:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575872.405791:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575872.405793:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393241984, portal 25
00000100:00000001:2.0:1788575872.405799:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575872.405800:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.405803:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00080000:3.0:1788575872.405803:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575872.405808:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575872.405810:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575872.405811:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1788575872.405814:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575872.405815:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575872.405819:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12803
00000400:00000200:3.0:1788575872.405825:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575872.405826:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575872.405828:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575872.405830:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405834:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575872.405837:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575872.405840:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575872.405840:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575872.405843:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575872.405844:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575872.405846:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575872.405847:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575872.405848:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575872.405851:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575872.405852:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:2.0:1788575872.405853:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000040:3.0:1788575872.405855:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405858:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:2.0:1788575872.405860:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393241984, offset 0
00000100:00000040:3.0:1788575872.405860:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405863:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575872.405866:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575872.405868:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405870:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575872.405871:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575872.405872:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575872.405874:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575872.405875:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575872.405876:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575872.405877:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575872.405878:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575872.405880:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788575872.405881:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575872.405883:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575872.405884:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575872.405891:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575872.405893:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575872.405894:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575872.405898:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000100:00000001:3.0:1788575872.405901:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575872.405905:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575872.405912:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575872.405914:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575872.405917:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575872.405926:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575872.405931:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575872.405934:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575872.405936:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575872.405940:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788575872.405944:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1788575872.405946:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12813)
00000400:00000200:2.0:1788575872.405949:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575872.405953:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575872.405956:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575872.405959:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575872.405961:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331146:331146:256:4294967295] 192.168.202.110@tcp LPNI seq info [331103:331103:8:4294967295]
00000400:00000200:2.0:1788575872.405964:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575872.405971:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575872.405975:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575872.405979:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575872.405983:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575872.405986:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788575872.405990:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575872.405993:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788575872.405995:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575872.405997:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575872.406002:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575872.406005:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575872.406009:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575872.406011:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575872.406015:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575872.406019:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406029:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575872.406032:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406039:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.406041:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575872.406047:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575872.406051:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575872.406054:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307476921
00000100:00000010:2.0:1788575872.406058:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ccc0.
00000100:00000001:2.0:1788575872.406064:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575872.406070:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307476921
00000100:00000010:2.0:1788575872.406073:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c4e0.
00000100:00000001:2.0:1788575872.406075:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575872.406077:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575872.406079:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.406080:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575872.406084:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575872/real 1788575872]  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575872.406091:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575872.406093:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575872.406095:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575872.406097:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575872.406098:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575872.406099:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575872.406102:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575872.406107:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406133:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575872.406134:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406136:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575872.406138:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575872.406142:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406150:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575872.406154:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406161:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575872.406164:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406165:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575872.406167:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406169:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575872.406171:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575872.406174:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575872.406178:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575872.406180:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575872.406183:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575872.406188:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393241984:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575872.406193:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406195:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575872.406196:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393241984/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406200:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575872.406202:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575872.406204:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575872.406206:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575872.406207:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575872.406208:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575872.406210:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575872.406212:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575872.406214:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575872.406215:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575872.406217:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788575872.406219:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575872.406220:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406221:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406226:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575872.406228:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.406229:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575872.406230:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575872.406231:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575872.406233:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406237:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575872.406238:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575872.406240:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575872.406241:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575872.406242:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575872.406244:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242112:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575872.406245:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575872.406247:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575872.406248:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575872.406249:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575872.406251:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575872.406252:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575872.406255:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575872.406258:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575872.406261:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575872.406264:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393242112, portal 10
00000100:00000001:2.0:1788575872.406266:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575872.406267:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:2.0:1788575872.406273:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406279:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575872.406281:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575872.406284:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575872.406286:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393242112, offset 0
00000400:00000200:2.0:1788575872.406290:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575872.406294:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575872.406298:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575872.406301:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575872.406303:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575872.406304:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575872.406306:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575872.406308:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331147:331147:256:4294967295] 192.168.202.110@tcp LPNI seq info [331104:331104:8:4294967295]
00000400:00000200:2.0:1788575872.406311:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575872.406315:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575872.406318:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575872.406320:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575872.406323:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575872.406325:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788575872.406329:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575872.406332:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788575872.406336:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575872.406337:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575872.406340:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575872.406343:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575872.406346:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575872.406348:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575872.406350:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575872.406354:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406407:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575872.406410:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406414:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.406415:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575872.406418:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575872.406421:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575872.406423:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575872.406425:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307476007
00000100:00000001:2.0:1788575872.406426:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307476921
00000100:00000001:2.0:1788575872.406428:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307476007
00000100:00000001:2.0:1788575872.406429:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307476921
00000100:00000001:2.0:1788575872.406431:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575872.406432:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.406433:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575872.406435:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575872/real 1788575872]  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575872.406439:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575872.406440:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575872.406441:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575872.406443:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575872.406444:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575872.406445:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575872.406447:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575872.406452:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406455:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575872.406456:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406457:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575872.406458:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575872.406460:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406463:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575872.406465:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406469:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575872.406470:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406471:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575872.406472:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406473:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575872.406474:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575872.406476:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575872.406478:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575872.406479:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575872.406482:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575872.406486:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242112:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575872.406488:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575872.406490:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406497:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575872.406498:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575872.406499:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575872.406500:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575872.406501:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575872.406503:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242240:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575872.406505:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575872.406506:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575872.406507:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575872.406508:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575872.406510:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575872.406511:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575872.406513:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575872.406514:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575872.406516:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575872.406518:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393242240, portal 4
00000100:00000001:2.0:1788575872.406519:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575872.406520:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788575872.406524:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406527:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575872.406528:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575872.406530:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575872.406531:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393242240, offset 0
00000400:00000200:2.0:1788575872.406533:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575872.406535:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575872.406538:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575872.406540:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575872.406541:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575872.406543:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575872.406544:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575872.406546:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331148:331148:256:4294967295] 192.168.202.110@tcp LPNI seq info [331105:331105:8:4294967295]
00000400:00000200:2.0:1788575872.406549:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575872.406553:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575872.406556:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575872.406558:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575872.406560:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575872.406562:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788575872.406564:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575872.406566:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788575872.406568:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575872.406569:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575872.406571:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575872.406573:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575872.406574:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575872.406575:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575872.406577:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575872.406579:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575897 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406583:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575872.406585:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575897 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406588:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.406589:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575872.406590:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575872.406592:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575872.406594:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575872.406595:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307476153
00000100:00000001:2.0:1788575872.406596:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307476921
00000100:00000001:2.0:1788575872.406598:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307476153
00000100:00000001:2.0:1788575872.406599:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307476921
00000100:00000001:2.0:1788575872.406600:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575872.406602:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.406603:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575872.406606:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575872/real 1788575872]  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575872.406613:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575872.406615:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575872.406616:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575872.406618:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575872.406619:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575872.406621:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575872.406624:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575872.406632:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406639:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575872.406640:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406642:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575872.406643:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575872.406647:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406653:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575872.406656:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406660:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575872.406662:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406663:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575872.406663:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406664:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575872.406665:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575872.406667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575872.406669:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575872.406670:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575872.406673:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575872.406677:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242240:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575872.406679:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406680:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575872.406682:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393242112/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406685:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575872.406686:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575872.406688:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575872.406690:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575872.406691:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575872.406693:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575872.406695:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575872.406696:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575872.406698:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575872.406699:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575872.406700:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575872.406703:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575872.406704:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406705:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575872.406709:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393242240/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575897 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575872.406715:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575872.406716:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575872.406718:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575872.406720:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575872.406721:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575872.406723:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575872.406725:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575872.406726:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:2.0:1788575872.406728:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575872.406729:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575872.406730:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575872.406733:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575872.406734:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406735:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575872.406738:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575872.406739:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575872.406740:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575872.406741:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575872.789507:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575872.789514:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575872.789520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575872.789522:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575872.789526:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575872.789528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575872.789569:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575872.789574:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575872.789577:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575872.917507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575872.917514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575872.917517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575872.917522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575872.917526:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575872.917529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575872.917532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575872.917535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575872.917538:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575873.365497:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307476921, at 4307476928
00000100:00000010:0.0:1788575873.365509:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000100:00000040:1.0:1788575873.366157:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307476921, at 4307476928
00000100:00000010:1.0:1788575873.366164:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00010000:00000001:3.0:1788575873.814199:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575873.814205:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575873.814210:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575873.814212:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575873.814215:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575873.814216:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575873.814267:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575873.814270:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575873.814272:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575873.941504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575873.941512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575873.941515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575873.941520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575873.941524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575873.941527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575873.941530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575873.941532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575873.941534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575874.838179:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575874.838185:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575874.838189:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575874.838190:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575874.838193:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575874.838194:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575874.838411:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575874.838415:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575874.838418:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575874.965504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575874.965512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575874.965514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575874.965520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575874.965524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575874.965527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575874.965530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575874.965533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575874.965535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575875.861510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575875.861517:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575875.861522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575875.861525:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575875.861527:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575875.861531:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575875.861533:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575875.861535:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575875.861538:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575875.861540:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575875.861543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575875.861544:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575875.861545:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575875.861547:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575875.861548:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575875.861550:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575875.861551:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575875.861553:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575875.861605:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575875.861608:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575875.861610:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575875.989525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575875.989533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575875.989535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575875.989540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575875.989544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575875.989548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575875.989550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575875.989552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575875.989555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575876.885539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575876.885546:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575876.885552:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575876.885553:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575876.885557:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575876.885559:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575876.885665:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575876.885669:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575876.885672:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575877.013502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575877.013508:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575877.013510:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575877.013513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575877.013516:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575877.013519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575877.013521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575877.013523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575877.013525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575877.525492:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575877.525503:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575877.525505:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575877.525510:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575877.525521:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575877.525529:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12808
00000400:00000200:3.0:1788575877.525537:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575877.525543:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575877.525546:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575877.525550:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525552:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575877.525554:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525557:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575877.525560:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575877.525562:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575877.525565:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575877.525568:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575877.525570:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575877.525573:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575877.525576:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525578:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575877.525580:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525582:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575877.525585:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575877.525588:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525591:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575877.525594:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575877.525597:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575877.525600:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575877.525602:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575877.525604:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575877.525613:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000020:00000040:3.0:1788575877.525617:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575877.525620:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575877.525622:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575877.525625:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575877.525627:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575877.525631:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575877.525639:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:3.0:1788575877.525644:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575877.525651:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575877.525658:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575877.525661:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575877.525664:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575877.525684:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575877.525687:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575877.525690:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575877.525692:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575877.525693:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575877.525697:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575877.525700:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12808
00000400:00000200:3.0:1788575877.525705:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575877.525706:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:3.0:1788575877.525710:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575877.525713:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575877.525716:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575877.525717:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575877.525719:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575877.525721:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575877.525727:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575877.525727:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525731:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575877.525736:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575877.525739:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575877.525742:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575877.525745:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:3.0:1788575877.525746:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575877.525748:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1788575877.525750:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575877.525754:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:3.0:1788575877.525755:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575877.525758:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1788575877.525762:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575877.525763:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575877.525767:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00100000:2.0:1788575877.525771:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393242368:192.168.202.110@tcp:250:kworker.0
00000100:00000040:3.0:1788575877.525773:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525778:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575877.525779:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1788575877.525783:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575877.525784:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:3.0:1788575877.525787:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575877.525788:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575877.525792:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575877.525793:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575877.525796:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:2.0:1788575877.525797:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575877.525799:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575877.525801:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575877.525804:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575877.525807:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575877.525808:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575877.525811:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575877.525813:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575877.525814:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575877.525819:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000400:00000010:2.0:1788575877.525820:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000020:00000040:3.0:1788575877.525822:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575877.525825:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575877.525826:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575877.525827:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1788575877.525828:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
02000000:00000001:3.0:1788575877.525829:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575877.525832:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575877.525834:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393242368, portal 25
02000000:00000010:3.0:1788575877.525837:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8c00.
00000100:00000001:2.0:1788575877.525838:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575877.525842:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.525844:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00100000:3.0:1788575877.525848:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788575877.525852:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575877.525865:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575877.525867:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575877.525868:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575877.525872:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575877.525875:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575877.525877:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575877.525882:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393242368, offset 0
00000100:00000001:3.0:1788575877.525885:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575877.525888:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575877.525890:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575877.525892:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575877.525894:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575877.525895:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575877.525897:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575877.525899:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575877.525902:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12808
00000400:00000200:2.0:1788575877.525906:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575877.525910:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575877.525912:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575877.525915:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575877.525918:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575877.525920:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331149:331149:256:4294967295] 192.168.202.110@tcp LPNI seq info [331106:331106:8:4294967295]
00000400:00000200:2.0:1788575877.525923:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575877.525934:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575877.525942:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575877.525944:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1788575877.525947:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1788575877.525951:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575877.525953:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525956:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000010:2.0:1788575877.525957:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000100:00000040:3.0:1788575877.525957:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525961:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575877.525963:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575877.525964:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575877.525966:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.0:1788575877.525967:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1788575877.525968:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:2.0:1788575877.525971:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:3.0:1788575877.525971:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575877.525972:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575877.525979:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.1:1788575877.525980:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575877.525981:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1788575877.525984:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00000040:3.0:1788575877.525986:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575877.525989:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000100:2.0:1788575877.525992:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575877.525993:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000010:2.0:1788575877.525996:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000100:00080000:3.0:1788575877.525998:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575877.526002:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1788575877.526003:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575877.526019:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575877.526020:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:3.0:1788575877.526023:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575877.526026:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
02000000:00000001:3.0:1788575877.526026:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575877.526029:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:3.0:1788575877.526030:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575877.526033:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575877.526035:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000008:00000001:3.0:1788575877.526036:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575877.526039:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575877.526041:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1788575877.526043:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575877.526047:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575877.526051:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575877.526053:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575877.526057:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575877.526059:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575877.526060:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1788575877.526060:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575877.526067:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575902 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788575877.526067:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000100:00000001:3.0:1788575877.526072:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575877.526080:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575877.526082:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.526087:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575877.526091:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575877.526092:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.526096:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575877.526096:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575877.526103:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575877.526108:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307482041
00000100:00000001:3.0:1788575877.526108:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575877.526113:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:2.0:1788575877.526116:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c600.
00000100:00000040:3.0:1788575877.526116:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575877.526122:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12818)
00000100:00000001:2.0:1788575877.526124:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575877.526130:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307482041
00000100:00000010:2.0:1788575877.526132:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cf00.
00000100:00000001:2.0:1788575877.526135:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575877.526144:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575877.526145:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.526149:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575877.526153:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575877/real 1788575877]  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575877.526160:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575877.526161:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575877.526162:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575877.526164:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575877.526165:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575877.526167:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575877.526170:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575877.526175:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526179:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575877.526180:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526181:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575877.526182:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575877.526185:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526189:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575877.526191:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526196:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575877.526197:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526198:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575877.526199:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526200:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575877.526202:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575877.526203:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575877.526205:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575877.526207:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575877.526209:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575877.526213:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393242368:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575877.526216:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526218:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575877.526219:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393242368/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526227:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575877.526229:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575877.526231:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575877.526233:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575877.526234:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575877.526235:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575877.526236:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575877.526238:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:2.0:1788575877.526240:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575877.526242:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575877.526243:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:2.0:1788575877.526245:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575877.526246:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526247:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526253:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575877.526254:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.526255:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575877.526257:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575877.526258:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575877.526260:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526264:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575877.526265:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575877.526266:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575877.526267:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575877.526268:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575877.526270:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242496:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575877.526272:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575877.526273:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575877.526274:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575877.526275:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575877.526277:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575877.526278:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575877.526280:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575877.526282:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575877.526284:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575877.526286:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393242496, portal 10
00000100:00000001:2.0:1788575877.526287:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575877.526288:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:2.0:1788575877.526292:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526296:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575877.526297:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575877.526298:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575877.526300:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393242496, offset 0
00000400:00000200:2.0:1788575877.526302:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575877.526304:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575877.526306:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575877.526308:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575877.526310:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575877.526311:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575877.526313:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575877.526315:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331150:331150:256:4294967295] 192.168.202.110@tcp LPNI seq info [331107:331107:8:4294967295]
00000400:00000200:2.0:1788575877.526317:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575877.526329:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575877.526336:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575877.526340:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575877.526343:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575877.526346:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788575877.526351:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575877.526353:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1788575877.526357:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575877.526358:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575877.526361:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575877.526364:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575877.526367:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575877.526369:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575877.526372:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575877.526374:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526379:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575877.526381:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575902 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526385:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.526386:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575877.526388:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575877.526409:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575877.526413:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575877.526415:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307481127
00000100:00000001:2.0:1788575877.526418:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307482041
00000100:00000001:2.0:1788575877.526420:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307481127
00000100:00000001:2.0:1788575877.526422:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307482041
00000100:00000001:2.0:1788575877.526424:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575877.526426:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.526428:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575877.526431:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575877/real 1788575877]  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575877.526436:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575877.526437:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575877.526439:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575877.526440:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575877.526441:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575877.526443:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575877.526445:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575877.526450:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526457:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575877.526458:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526459:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575877.526460:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575877.526462:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526466:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575877.526468:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526472:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575877.526473:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526474:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575877.526474:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526475:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575877.526477:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575877.526478:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575877.526480:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575877.526481:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575877.526483:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575877.526487:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242496:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575877.526489:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575877.526491:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526495:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575877.526496:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575877.526498:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575877.526499:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575877.526500:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575877.526502:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575877.526504:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575877.526505:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575877.526506:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575877.526507:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575877.526509:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575877.526510:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575877.526512:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575877.526514:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575877.526516:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575877.526518:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393242624, portal 4
00000100:00000001:2.0:1788575877.526519:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575877.526520:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788575877.526524:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526527:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575877.526529:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575877.526530:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575877.526532:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393242624, offset 0
00000400:00000200:2.0:1788575877.526534:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575877.526536:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575877.526547:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575877.526549:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575877.526552:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575877.526554:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575877.526556:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575877.526559:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331151:331151:256:4294967295] 192.168.202.110@tcp LPNI seq info [331108:331108:8:4294967295]
00000400:00000200:2.0:1788575877.526563:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575877.526569:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575877.526574:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575877.526577:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575877.526580:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575877.526583:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1788575877.526587:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575877.526589:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1788575877.526593:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575877.526595:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575877.526597:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575877.526600:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575877.526602:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575877.526603:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575877.526604:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575877.526613:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575902 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526619:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575877.526621:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575902 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526625:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.526626:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575877.526628:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575877.526630:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575877.526632:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575877.526634:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307481273
00000100:00000001:2.0:1788575877.526635:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307482041
00000100:00000001:2.0:1788575877.526636:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307481273
00000100:00000001:2.0:1788575877.526638:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307482041
00000100:00000001:2.0:1788575877.526639:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575877.526640:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.526641:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575877.526643:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575877/real 1788575877]  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575877.526648:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575877.526649:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575877.526650:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575877.526651:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575877.526652:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575877.526653:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575877.526655:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575877.526660:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526663:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575877.526664:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526665:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575877.526665:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575877.526667:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526671:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575877.526673:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526676:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575877.526678:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526679:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575877.526679:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526681:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575877.526682:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575877.526683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575877.526685:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575877.526686:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575877.526688:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575877.526693:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575877.526695:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526696:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575877.526697:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393242496/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526701:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575877.526702:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575877.526703:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575877.526704:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575877.526705:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575877.526706:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575877.526707:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575877.526708:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:2.0:1788575877.526709:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575877.526710:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575877.526710:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575877.526712:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575877.526713:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526714:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575877.526716:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393242624/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575902 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575877.526719:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575877.526720:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575877.526721:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575877.526722:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575877.526722:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575877.526723:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575877.526724:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575877.526725:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:2.0:1788575877.526727:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575877.526729:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575877.526730:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575877.526732:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575877.526733:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526734:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575877.526738:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575877.526739:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575877.526741:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575877.526743:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575877.909526:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575877.909532:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575877.909537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575877.909539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575877.909543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575877.909544:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575877.909694:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575877.909697:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575877.909701:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575878.037494:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575878.037500:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575878.037501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575878.037505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575878.037509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575878.037512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575878.037514:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575878.037515:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575878.037518:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575878.485498:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307482041, at 4307482048
00000100:00000040:0.0:1788575878.485512:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307482041, at 4307482048
00000100:00000010:1.0:1788575878.485513:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000100:00000010:0.0:1788575878.485523:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00010000:00000001:3.0:1788575878.933516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575878.933523:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575878.933525:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575878.933528:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575878.933533:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575878.933535:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575878.933537:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575878.933539:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575878.933542:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575878.933544:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575878.933547:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575878.933548:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575878.933665:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575878.933668:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575878.933670:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575879.061543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575879.061560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575879.061562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575879.061567:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575879.061572:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575879.061576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575879.061579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575879.061581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575879.061584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575879.957519:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575879.957526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575879.957532:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575879.957533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575879.957537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575879.957539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575879.957572:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575879.957576:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575879.957580:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575880.085533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575880.085539:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575880.085541:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575880.085546:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575880.085549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575880.085552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575880.085554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575880.085556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575880.085558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575880.981666:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575880.981672:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575880.981678:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575880.981679:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575880.981683:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575880.981684:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575880.981738:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575880.981741:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575880.981744:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575881.109503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575881.109517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575881.109520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575881.109526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575881.109530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575881.109533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575881.109536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575881.109538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575881.109541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575882.005520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575882.005526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575882.005531:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575882.005533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575882.005537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575882.005538:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575882.005639:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575882.005647:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575882.005650:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575882.133546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575882.133554:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575882.133556:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575882.133561:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575882.133565:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575882.133569:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575882.133572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575882.133574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575882.133577:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575882.645479:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575882.645487:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575882.645489:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575882.645492:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575882.645496:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575882.645502:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12813
00000400:00000200:2.0:1788575882.645516:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575882.645523:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575882.645526:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575882.645530:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645533:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575882.645535:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645538:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575882.645542:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575882.645544:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575882.645547:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575882.645551:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575882.645553:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575882.645556:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575882.645558:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645559:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575882.645565:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645567:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575882.645570:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575882.645573:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645574:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575882.645577:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575882.645579:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575882.645581:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575882.645582:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575882.645584:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575882.645593:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02680.
00000020:00000040:2.0:1788575882.645596:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575882.645600:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575882.645601:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575882.645604:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575882.645607:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575882.645616:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575882.645624:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11400.
00000100:00000001:2.0:1788575882.645631:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575882.645640:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575882.645648:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575882.645652:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575882.645656:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000450d1599] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575882.645680:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575882.645683:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575882.645687:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575882.645688:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575882.645690:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575882.645692:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575882.645694:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12813
00000400:00000200:2.0:1788575882.645697:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575882.645700:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575882.645702:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575882.645705:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645707:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575882.645709:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645712:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:3.0:1788575882.645712:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788575882.645715:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575882.645719:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:3.0:1788575882.645719:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1788575882.645723:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575882.645724:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1788575882.645725:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575882.645730:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575882.645731:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575882.645733:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575882.645737:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645739:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575882.645743:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645747:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575882.645753:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575882.645759:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575882.645759:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575882.645763:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.645764:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575882.645766:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575882.645770:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575882.645773:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575882.645774:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575882.645778:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575882.645780:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575882.645782:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:3.0:1788575882.645787:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393242752:192.168.202.110@tcp:250:kworker.0
00000008:00100000:2.0:1788575882.645788:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575882.645795:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575882.645796:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.645800:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:2.0:1788575882.645801:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00700.
00000020:00000040:2.0:1788575882.645805:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788575882.645808:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575882.645808:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575882.645811:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575882.645812:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575882.645813:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575882.645816:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575882.645818:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1788575882.645820:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575882.645827:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
02000000:00000010:3.0:1788575882.645827:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575882.645832:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575882.645834:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575882.645840:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1788575882.645842:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1788575882.645847:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:2.0:1788575882.645854:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575882.645856:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575882.645860:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e69941e] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788575882.645861:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575882.645869:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393242752, portal 25
00000100:00000001:2.0:1788575882.645871:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.645872:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575882.645875:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00080000:2.0:1788575882.645876:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575882.645880:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575882.645883:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:3.0:1788575882.645884:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575882.645885:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575882.645890:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575882.645895:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12813
00000100:00000001:3.0:1788575882.645896:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:2.0:1788575882.645911:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575882.645914:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575882.645916:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575882.645917:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645918:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575882.645920:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645923:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:3.0:1788575882.645926:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:2.0:1788575882.645927:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575882.645930:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000010:3.0:1788575882.645931:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000020:00000001:2.0:1788575882.645932:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575882.645935:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575882.645937:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575882.645938:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:3.0:1788575882.645941:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393242752, offset 0
00000100:00000040:2.0:1788575882.645942:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645946:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575882.645948:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645950:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:3.0:1788575882.645953:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:2.0:1788575882.645954:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788575882.645958:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575882.645960:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575882.645961:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1788575882.645963:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575882.645964:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575882.645968:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575882.645969:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1788575882.645972:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575882.645977:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00100000:2.0:1788575882.645981:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:3.0:1788575882.645982:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00000001:2.0:1788575882.645984:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575882.645986:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575882.645987:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000010:2.0:1788575882.645991:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000400:00000200:3.0:1788575882.645991:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:2.0:1788575882.645995:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:3.0:1788575882.645998:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331152:331152:256:4294967295] 192.168.202.110@tcp LPNI seq info [331109:331109:8:4294967295]
02000000:00000001:2.0:1788575882.645999:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575882.646001:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575882.646004:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575882.646007:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1788575882.646007:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575882.646014:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575882.646019:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000010:2.0:1788575882.646020:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000001:2.0:1788575882.646025:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575882.646029:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00100000:2.0:1788575882.646033:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:3.0:1788575882.646036:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000100:00000001:2.0:1788575882.646047:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:3.0:1788575882.646049:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575882.646050:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575882.646055:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000100:00000040:2.0:1788575882.646056:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1788575882.646063:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575882.646068:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000100:00000001:2.0:1788575882.646073:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1788575882.646076:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575882.646077:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:3.0:1788575882.646078:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000100:00000040:2.0:1788575882.646083:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1788575882.646086:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1788575882.646087:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12823)
00000400:00000200:3.0:1788575882.646092:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575882.646096:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575882.646097:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575882.646101:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575882.646105:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575882.646114:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646119:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.646121:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575882.646127:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575882.646131:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575882.646133:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307487161
00000100:00000010:3.0:1788575882.646139:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788575882.646145:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575882.646151:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307487161
00000100:00000010:3.0:1788575882.646154:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788575882.646157:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575882.646160:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575882.646163:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.646165:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575882.646169:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575882/real 1788575882]  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575882.646181:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575882.646182:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575882.646184:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575882.646186:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575882.646187:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575882.646189:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575882.646191:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575882.646196:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646200:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575882.646202:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646203:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575882.646205:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575882.646213:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646218:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575882.646221:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646228:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575882.646231:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646233:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575882.646234:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646236:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575882.646238:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575882.646240:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575882.646244:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575882.646246:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575882.646250:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575882.646271:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393242752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575882.646275:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646277:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575882.646280:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393242752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646286:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575882.646288:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575882.646291:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575882.646294:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575882.646296:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575882.646297:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575882.646299:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575882.646302:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575882.646305:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575882.646307:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575882.646309:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575882.646312:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575882.646313:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646314:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646319:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575882.646320:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.646322:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575882.646323:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575882.646324:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575882.646327:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646331:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575882.646332:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575882.646333:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575882.646335:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575882.646336:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575882.646338:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575882.646340:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575882.646341:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575882.646342:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575882.646343:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575882.646345:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575882.646347:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575882.646349:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575882.646351:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575882.646354:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575882.646356:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393242880, portal 10
00000100:00000001:3.0:1788575882.646357:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575882.646358:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:3.0:1788575882.646363:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646367:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575882.646370:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575882.646372:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575882.646375:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393242880, offset 0
00000400:00000200:3.0:1788575882.646379:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575882.646383:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575882.646386:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575882.646413:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575882.646417:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575882.646420:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575882.646422:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575882.646425:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331153:331153:256:4294967295] 192.168.202.110@tcp LPNI seq info [331110:331110:8:4294967295]
00000400:00000200:3.0:1788575882.646430:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575882.646451:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575882.646455:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575882.646459:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575882.646463:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575882.646466:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1788575882.646470:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575882.646473:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1788575882.646477:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575882.646479:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575882.646482:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575882.646485:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575882.646488:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575882.646489:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575882.646491:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575882.646495:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646501:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575882.646503:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646508:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.646510:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575882.646512:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575882.646516:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575882.646519:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575882.646520:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307486247
00000100:00000001:3.0:1788575882.646523:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307487161
00000100:00000001:3.0:1788575882.646525:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307486247
00000100:00000001:3.0:1788575882.646527:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307487161
00000100:00000001:3.0:1788575882.646529:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575882.646531:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.646533:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575882.646536:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575882/real 1788575882]  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575882.646543:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575882.646544:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575882.646559:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575882.646562:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575882.646563:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575882.646565:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575882.646568:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575882.646575:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646581:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575882.646582:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646584:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575882.646585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575882.646588:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646594:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575882.646597:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646604:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575882.646606:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646607:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575882.646608:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646610:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575882.646612:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575882.646614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575882.646617:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575882.646618:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575882.646622:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575882.646629:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393242880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575882.646634:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575882.646637:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646643:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575882.646645:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575882.646646:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575882.646648:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575882.646650:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575882.646653:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243008:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575882.646655:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575882.646656:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575882.646657:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575882.646658:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575882.646661:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575882.646662:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575882.646665:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575882.646666:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575882.646668:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575882.646670:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393243008, portal 4
00000100:00000001:3.0:1788575882.646671:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575882.646672:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575882.646677:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646683:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575882.646685:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575882.646687:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575882.646689:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393243008, offset 0
00000400:00000200:3.0:1788575882.646693:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575882.646696:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575882.646700:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575882.646702:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575882.646705:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575882.646707:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575882.646709:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575882.646712:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331154:331154:256:4294967295] 192.168.202.110@tcp LPNI seq info [331111:331111:8:4294967295]
00000400:00000200:3.0:1788575882.646716:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575882.646722:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575882.646727:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575882.646730:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575882.646733:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575882.646735:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1788575882.646739:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575882.646741:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1788575882.646745:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575882.646746:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575882.646749:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575882.646752:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575882.646755:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575882.646757:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575882.646759:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575882.646762:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646770:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575882.646773:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646780:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.646781:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575882.646784:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575882.646786:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575882.646788:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575882.646789:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307486393
00000100:00000001:3.0:1788575882.646791:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307487161
00000100:00000001:3.0:1788575882.646793:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307486393
00000100:00000001:3.0:1788575882.646795:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307487161
00000100:00000001:3.0:1788575882.646797:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575882.646799:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.646801:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575882.646805:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575882/real 1788575882]  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575882.646812:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575882.646814:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575882.646815:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575882.646818:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575882.646819:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575882.646821:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575882.646823:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575882.646830:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646835:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575882.646836:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646838:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575882.646839:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575882.646842:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646847:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575882.646851:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646856:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575882.646858:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646860:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575882.646861:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646863:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575882.646865:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575882.646867:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575882.646870:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575882.646873:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575882.646876:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575882.646883:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243008:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575882.646886:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646887:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575882.646890:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393242880/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646895:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575882.646896:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575882.646909:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575882.646913:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575882.646914:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575882.646915:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575882.646917:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575882.646918:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788575882.646920:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575882.646921:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575882.646922:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575882.646925:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575882.646926:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646927:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575882.646930:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393243008/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575882.646935:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575882.646936:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575882.646937:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575882.646939:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575882.646940:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575882.646941:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575882.646943:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575882.646944:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575882.646946:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575882.646946:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575882.646948:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575882.646949:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575882.646950:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646951:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575882.646955:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575882.646956:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575882.646958:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575882.646959:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575883.029512:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575883.029518:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575883.029524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575883.029526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575883.029529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575883.029531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575883.029656:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575883.029660:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575883.029663:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575883.157539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575883.157544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575883.157546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575883.157551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575883.157554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575883.157556:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575883.157558:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575883.157559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575883.157562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575883.605509:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307487161, at 4307487168
00000100:00000040:1.0:1788575883.605509:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307487161, at 4307487168
00000100:00000010:0.0:1788575883.605525:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000010:1.0:1788575883.605527:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00010000:00000001:3.0:1788575884.053521:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575884.053528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575884.053534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575884.053535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575884.053538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575884.053539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575884.053601:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575884.053605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575884.053609:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575884.182966:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575884.182974:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575884.182976:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575884.182981:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575884.182984:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575884.182987:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575884.182989:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575884.182991:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575884.182993:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575885.077502:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575885.077507:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575885.077513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575885.077514:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575885.077518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575885.077519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575885.077567:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575885.077571:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575885.077574:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575885.205559:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575885.205566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575885.205568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575885.205573:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575885.205577:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575885.205580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575885.205582:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575885.205585:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575885.205599:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575886.101524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575886.101530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575886.101536:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575886.101539:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575886.101541:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575886.101545:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575886.101548:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575886.101551:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575886.101553:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575886.101554:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575886.101557:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575886.101558:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575886.101559:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575886.101560:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575886.101561:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575886.101562:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575886.101563:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575886.101564:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575886.101627:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575886.101631:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575886.101641:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575886.229498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575886.229506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575886.229508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575886.229513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575886.229517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575886.229520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575886.229523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575886.229525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575886.229528:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575887.125485:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575887.125491:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575887.125497:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575887.125499:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575887.125502:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575887.125503:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575887.125557:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575887.125561:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575887.125564:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575887.253543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575887.253551:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575887.253554:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575887.253560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575887.253564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575887.253568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575887.253571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575887.253573:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575887.253577:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575887.765492:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575887.765499:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575887.765501:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575887.765504:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575887.765507:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575887.765512:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12818
00000400:00000200:2.0:1788575887.765523:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575887.765528:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575887.765530:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575887.765532:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765534:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575887.765536:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765539:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575887.765543:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575887.765545:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575887.765548:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575887.765551:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575887.765553:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575887.765555:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575887.765558:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765560:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575887.765562:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765564:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575887.765568:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575887.765571:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765573:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575887.765576:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575887.765579:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575887.765582:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575887.765584:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575887.765586:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575887.765593:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03480.
00000020:00000040:2.0:1788575887.765596:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575887.765599:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575887.765600:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575887.765603:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575887.765605:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575887.765608:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575887.765613:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000100:00000001:2.0:1788575887.765617:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575887.765624:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575887.765631:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575887.765635:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575887.765638:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9656041] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575887.765653:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575887.765656:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575887.765658:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575887.765659:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575887.765660:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575887.765662:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575887.765663:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12818
00000400:00000200:2.0:1788575887.765666:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575887.765669:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575887.765671:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575887.765673:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765675:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575887.765676:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765678:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575887.765679:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575887.765680:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575887.765681:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575887.765682:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1788575887.765683:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575887.765686:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788575887.765688:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:3.0:1788575887.765689:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1788575887.765690:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765693:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1788575887.765693:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575887.765697:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765699:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575887.765701:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788575887.765704:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:3.0:1788575887.765704:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575887.765708:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575887.765712:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575887.765718:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575887.765719:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1788575887.765722:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575887.765723:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575887.765726:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1788575887.765728:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:2.0:1788575887.765737:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1788575887.765740:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:2.0:1788575887.765741:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575887.765744:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575887.765746:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575887.765750:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393243136:192.168.202.110@tcp:250:kworker.0
00000100:00000010:2.0:1788575887.765752:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01180.
00000100:00000001:3.0:1788575887.765754:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1788575887.765757:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575887.765759:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575887.765762:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575887.765764:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575887.765765:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575887.765766:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575887.765767:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575887.765768:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575887.765774:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575887.765776:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000100:00000001:2.0:1788575887.765780:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575887.765782:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575887.765786:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575887.765790:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575887.765806:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575887.765814:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1788575887.765815:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000100:00000001:2.0:1788575887.765816:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575887.765824:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b74282d7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788575887.765826:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575887.765836:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575887.765839:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575887.765843:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000200:3.0:1788575887.765844:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393243136, portal 25
00000100:00000001:2.0:1788575887.765845:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575887.765849:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575887.765850:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575887.765854:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000001:2.0:1788575887.765855:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575887.765859:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12818
00000100:00000040:3.0:1788575887.765865:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575887.765866:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575887.765872:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575887.765875:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575887.765880:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765883:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575887.765885:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765886:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575887.765887:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575887.765888:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575887.765891:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575887.765892:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:3.0:1788575887.765892:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:2.0:1788575887.765896:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575887.765899:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575887.765900:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788575887.765903:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765905:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1788575887.765906:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000040:2.0:1788575887.765907:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765911:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575887.765912:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000200:3.0:1788575887.765913:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393243136, offset 0
00000100:00080000:2.0:1788575887.765915:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765919:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575887.765920:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575887.765922:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575887.765925:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575887.765925:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765928:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575887.765930:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575887.765932:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575887.765933:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575887.765936:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000400:00000200:3.0:1788575887.765936:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1788575887.765940:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575887.765942:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1788575887.765944:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:2.0:1788575887.765945:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575887.765948:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575887.765950:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575887.765953:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575887.765953:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000010:2.0:1788575887.765956:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000400:00000200:3.0:1788575887.765959:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1788575887.765961:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575887.765966:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00100000:2.0:1788575887.765967:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1788575887.765975:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788575887.765976:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575887.765979:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575887.765983:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575887.765985:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331155:331155:256:4294967295] 192.168.202.110@tcp LPNI seq info [331112:331112:8:4294967295]
00000100:00000001:2.0:1788575887.765993:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575887.765996:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575887.765998:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788575887.766000:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575887.766005:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12829)
00000400:00000200:3.0:1788575887.766013:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575887.766019:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575887.766025:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575887.766030:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575887.766034:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575887.766039:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575887.766042:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575887.766046:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575887.766048:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575887.766054:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575887.766059:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575887.766064:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575887.766067:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575887.766070:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575887.766074:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766083:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575887.766086:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766092:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575887.766094:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575887.766101:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575887.766105:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575887.766107:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307492281
00000100:00000010:3.0:1788575887.766113:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788575887.766119:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788575887.766125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307492281
00000100:00000010:3.0:1788575887.766128:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788575887.766131:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575887.766134:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575887.766136:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575887.766139:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575887.766143:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575887/real 1788575887]  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575887.766154:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575887.766155:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575887.766157:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575887.766160:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575887.766161:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575887.766164:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575887.766167:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575887.766175:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766182:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575887.766183:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766185:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575887.766188:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575887.766191:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766197:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575887.766201:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766208:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575887.766210:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766212:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575887.766213:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766215:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575887.766216:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575887.766219:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575887.766222:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575887.766224:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575887.766227:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575887.766232:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393243136:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575887.766236:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766237:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575887.766239:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393243136/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766242:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575887.766244:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575887.766246:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575887.766248:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575887.766249:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575887.766251:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575887.766252:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575887.766254:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:3.0:1788575887.766257:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575887.766258:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575887.766260:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:3.0:1788575887.766261:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575887.766262:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766263:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766269:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575887.766270:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575887.766271:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575887.766272:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575887.766274:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575887.766276:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766280:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575887.766281:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575887.766282:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575887.766283:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575887.766284:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575887.766286:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243264:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575887.766287:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575887.766288:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575887.766289:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575887.766290:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575887.766292:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575887.766294:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575887.766296:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575887.766298:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575887.766300:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575887.766302:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393243264, portal 10
00000100:00000001:3.0:1788575887.766303:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575887.766304:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000040:3.0:1788575887.766308:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766311:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575887.766313:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575887.766314:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575887.766315:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393243264, offset 0
00000400:00000200:3.0:1788575887.766318:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575887.766320:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575887.766322:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575887.766324:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575887.766326:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575887.766327:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575887.766329:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575887.766330:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331156:331156:256:4294967295] 192.168.202.110@tcp LPNI seq info [331113:331113:8:4294967295]
00000400:00000200:3.0:1788575887.766333:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575887.766337:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575887.766340:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575887.766342:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575887.766344:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575887.766346:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575887.766348:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575887.766350:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575887.766352:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575887.766353:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575887.766355:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575887.766357:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575887.766359:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575887.766360:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575887.766361:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575887.766363:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575887.766373:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766376:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575887.766377:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575887.766379:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575887.766381:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575887.766383:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575887.766384:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307491366
00000100:00000001:3.0:1788575887.766386:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307492281
00000100:00000001:3.0:1788575887.766388:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307491366
00000100:00000001:3.0:1788575887.766389:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307492281
00000100:00000001:3.0:1788575887.766391:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575887.766392:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575887.766409:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575887.766411:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575887/real 1788575887]  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575887.766416:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575887.766417:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575887.766418:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575887.766419:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575887.766420:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575887.766421:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575887.766423:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575887.766431:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766436:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575887.766438:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766440:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575887.766441:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575887.766444:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766450:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575887.766454:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766460:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575887.766463:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766464:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575887.766465:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766467:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575887.766469:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575887.766471:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575887.766475:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575887.766477:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575887.766481:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575887.766489:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243264:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575887.766493:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575887.766497:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766513:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575887.766514:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575887.766516:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575887.766518:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575887.766520:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575887.766523:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243392:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575887.766526:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575887.766528:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575887.766530:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575887.766532:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575887.766535:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575887.766537:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575887.766541:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575887.766543:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575887.766547:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575887.766549:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393243392, portal 4
00000100:00000001:3.0:1788575887.766551:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575887.766553:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788575887.766558:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766564:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575887.766567:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575887.766575:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575887.766578:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393243392, offset 0
00000400:00000200:3.0:1788575887.766582:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575887.766586:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575887.766589:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575887.766592:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575887.766594:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575887.766596:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575887.766599:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575887.766602:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331157:331157:256:4294967295] 192.168.202.110@tcp LPNI seq info [331114:331114:8:4294967295]
00000400:00000200:3.0:1788575887.766606:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575887.766612:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575887.766616:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575887.766620:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575887.766624:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575887.766627:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1788575887.766631:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575887.766634:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1788575887.766638:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575887.766640:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575887.766642:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575887.766646:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575887.766649:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575887.766651:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575887.766653:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575887.766656:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766664:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575887.766667:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766673:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575887.766675:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575887.766677:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575887.766680:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575887.766683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575887.766685:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307491513
00000100:00000001:3.0:1788575887.766688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307492281
00000100:00000001:3.0:1788575887.766689:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307491513
00000100:00000001:3.0:1788575887.766691:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307492281
00000100:00000001:3.0:1788575887.766694:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575887.766696:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575887.766698:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575887.766701:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575887/real 1788575887]  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575887.766708:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575887.766710:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575887.766711:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575887.766714:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575887.766715:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575887.766717:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575887.766720:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575887.766727:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766733:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575887.766734:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766736:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575887.766737:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575887.766741:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766747:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575887.766750:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766756:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575887.766759:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766760:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575887.766761:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766763:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575887.766764:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575887.766767:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575887.766770:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575887.766772:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575887.766776:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575887.766788:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243392:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575887.766791:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766808:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575887.766812:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393243264/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766819:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575887.766820:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575887.766822:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575887.766825:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575887.766826:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575887.766827:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575887.766829:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575887.766831:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:3.0:1788575887.766833:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575887.766834:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575887.766835:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:3.0:1788575887.766839:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575887.766840:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766841:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575887.766844:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393243392/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575887.766850:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575887.766851:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575887.766853:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575887.766855:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575887.766856:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575887.766857:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575887.766859:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575887.766861:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788575887.766862:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575887.766863:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575887.766865:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788575887.766867:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575887.766868:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766870:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575887.766874:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575887.766875:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575887.766877:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575887.766878:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575888.149512:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575888.149518:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575888.149523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575888.149524:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575888.149527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575888.149528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575888.149577:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575888.149581:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575888.149584:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575888.278336:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575888.278342:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575888.278344:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575888.278347:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575888.278349:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575888.278352:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575888.278354:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575888.278356:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575888.278358:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575888.725466:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307492281, at 4307492288
00000100:00000040:1.0:1788575888.725466:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307492281, at 4307492288
00000100:00000010:1.0:1788575888.725483:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000010:0.0:1788575888.725488:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788575889.173516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575889.173523:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575889.173525:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575889.173528:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575889.173533:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575889.173535:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575889.173536:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575889.173538:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575889.173541:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575889.173542:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575889.173549:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575889.173550:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575889.173597:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575889.173601:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575889.173609:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575889.301541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575889.301548:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575889.301550:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575889.301555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575889.301559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575889.301563:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575889.301566:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575889.301569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575889.301572:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575890.199192:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575890.199199:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575890.199205:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575890.199206:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575890.199209:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575890.199211:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575890.199277:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575890.199281:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575890.199284:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575890.325514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575890.325522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575890.325524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575890.325529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575890.325532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575890.325536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575890.325538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575890.325541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575890.325544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575891.221537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575891.221545:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575891.221551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575891.221552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575891.221555:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575891.221556:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788575891.221623:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1788575891.221627:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1788575891.221631:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575891.349588:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575891.349596:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575891.349598:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575891.349603:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575891.349607:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575891.349610:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575891.349613:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575891.349616:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575891.349619:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575892.246802:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575892.246809:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575892.246815:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575892.246816:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575892.246820:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575892.246822:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575892.246913:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575892.246916:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575892.246919:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575892.373512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575892.373520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575892.373522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575892.373527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575892.373530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575892.373534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575892.373536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575892.373539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575892.373542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575892.885502:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575892.885513:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575892.885516:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575892.885520:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575892.885524:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575892.885530:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12824
00000400:00000200:2.0:1788575892.885537:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575892.885541:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575892.885543:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575892.885545:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885548:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575892.885550:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885551:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575892.885554:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575892.885556:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575892.885558:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575892.885560:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575892.885561:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575892.885563:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575892.885564:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885566:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575892.885567:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885568:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575892.885570:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575892.885573:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885574:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575892.885576:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575892.885578:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575892.885580:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575892.885581:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575892.885583:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575892.885590:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00700.
00000020:00000040:2.0:1788575892.885593:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575892.885595:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575892.885596:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575892.885599:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575892.885601:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575892.885604:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575892.885610:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11400.
00000100:00000001:2.0:1788575892.885615:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575892.885622:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575892.885636:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575892.885640:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575892.885645:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e69941e] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575892.885666:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575892.885670:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575892.885673:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575892.885674:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575892.885676:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575892.885678:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575892.885681:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12824
00000400:00000200:2.0:1788575892.885684:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575892.885686:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1788575892.885714:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1788575892.885715:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1788575892.885718:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575892.885721:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575892.885722:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885725:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575892.885729:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885732:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:3.0:1788575892.885733:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1788575892.885737:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575892.885740:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575892.885744:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1788575892.885745:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1788575892.885748:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:3.0:1788575892.885749:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575892.885753:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:2.0:1788575892.885754:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:3.0:1788575892.885755:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1788575892.885759:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:3.0:1788575892.885759:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575892.885767:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393243520:192.168.202.110@tcp:250:kworker.0
00000100:00000040:2.0:1788575892.885768:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885772:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575892.885777:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575892.885778:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1788575892.885780:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:3.0:1788575892.885783:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:2.0:1788575892.885785:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:3.0:1788575892.885787:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575892.885791:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1788575892.885792:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788575892.885797:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575892.885799:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:3.0:1788575892.885800:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575892.885803:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575892.885806:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1788575892.885811:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788575892.885812:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575892.885817:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1788575892.885818:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000008:00100000:2.0:1788575892.885821:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788575892.885824:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1788575892.885825:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000001:2.0:1788575892.885826:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575892.885831:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02680.
00000100:00000200:3.0:1788575892.885832:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393243520, portal 25
00000020:00000040:2.0:1788575892.885834:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:3.0:1788575892.885835:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1788575892.885838:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1788575892.885838:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
02000000:00000001:2.0:1788575892.885841:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575892.885844:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575892.885846:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1788575892.885847:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575892.885848:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575892.885853:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:3.0:1788575892.885856:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1788575892.885858:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575892.885860:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575892.885864:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00100000:2.0:1788575892.885867:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1788575892.885868:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393243520, offset 0
00000100:00000001:2.0:1788575892.885880:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1788575892.885882:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885885:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575892.885892:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000450d1599] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575892.885892:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575892.885899:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575892.885908:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788575892.885911:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575892.885912:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1788575892.885917:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1788575892.885918:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575892.885922:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:2.0:1788575892.885926:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:3.0:1788575892.885926:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331158:331158:256:4294967295] 192.168.202.110@tcp LPNI seq info [331115:331115:8:4294967295]
00000100:00000001:2.0:1788575892.885931:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1788575892.885933:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575892.885936:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575892.885943:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:3.0:1788575892.885943:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1788575892.885949:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12824
00000800:00000200:3.0:1788575892.885952:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575892.885956:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:3.0:1788575892.885958:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000400:00000200:2.0:1788575892.885960:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575892.885963:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575892.885966:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575892.885969:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575892.885970:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788575892.885974:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:3.0:1788575892.885977:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1788575892.885979:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575892.885984:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575892.885987:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:3.1:1788575892.885988:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1788575892.885990:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575892.885994:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:3.1:1788575892.885995:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1788575892.885999:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000100:3.0:1788575892.886004:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:2.0:1788575892.886005:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000010:3.0:1788575892.886007:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000100:00000040:2.0:1788575892.886012:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575892.886015:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1788575892.886016:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575892.886021:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:3.0:1788575892.886025:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1788575892.886027:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575892.886031:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:3.0:1788575892.886032:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000100:00080000:2.0:1788575892.886036:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:3.0:1788575892.886036:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575892.886041:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:2.0:1788575892.886042:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575892.886046:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1788575892.886048:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575892.886050:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575892.886054:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575892.886058:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575892.886061:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575892.886063:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575892.886065:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1788575892.886066:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575892.886071:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1788575892.886072:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01f80.
00000020:00000040:2.0:1788575892.886077:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:3.0:1788575892.886083:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575892.886084:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1788575892.886086:0:2361:0:(events.c:69:request_out_callback()) Process leaving
02000000:00000001:2.0:1788575892.886088:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575892.886091:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1788575892.886093:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575892.886094:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575892.886098:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575892.886099:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1788575892.886104:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:3.0:1788575892.886105:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307497401
00000100:00000001:2.0:1788575892.886111:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575892.886113:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00100000:2.0:1788575892.886121:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575892.886122:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575892.886129:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575892.886130:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307497401
00000100:00000001:2.0:1788575892.886134:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575892.886136:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000040:2.0:1788575892.886139:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8c8ec7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886142:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788575892.886147:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575892.886152:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575892.886154:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575892.886156:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1788575892.886158:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1788575892.886163:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575892/real 1788575892]  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575892.886165:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788575892.886173:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12834)
00000400:00000001:3.0:1788575892.886177:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575892.886180:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575892.886182:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575892.886185:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575892.886187:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575892.886190:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575892.886192:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575892.886198:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886202:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575892.886203:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886205:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575892.886206:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575892.886208:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886213:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575892.886216:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886223:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575892.886226:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886228:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575892.886229:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886231:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575892.886233:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575892.886236:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575892.886240:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575892.886242:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575892.886252:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575892.886260:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393243520:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788575892.886265:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886266:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575892.886268:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393243520/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886272:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575892.886274:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575892.886276:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575892.886278:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575892.886280:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575892.886281:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788575892.886282:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575892.886284:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:3.0:1788575892.886287:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575892.886288:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575892.886289:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:3.0:1788575892.886291:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575892.886292:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886294:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886306:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575892.886308:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575892.886309:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575892.886311:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575892.886312:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575892.886314:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886318:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575892.886319:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575892.886321:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575892.886322:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575892.886323:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575892.886325:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575892.886327:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575892.886328:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575892.886329:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575892.886331:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575892.886333:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575892.886334:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575892.886337:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575892.886338:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575892.886341:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575892.886343:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393243648, portal 10
00000100:00000001:3.0:1788575892.886344:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575892.886345:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:3.0:1788575892.886350:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886354:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575892.886355:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575892.886356:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575892.886358:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393243648, offset 0
00000400:00000200:3.0:1788575892.886360:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575892.886362:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575892.886365:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575892.886366:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575892.886368:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575892.886370:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575892.886371:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575892.886373:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331159:331159:256:4294967295] 192.168.202.110@tcp LPNI seq info [331116:331116:8:4294967295]
00000400:00000200:3.0:1788575892.886376:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575892.886380:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575892.886383:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575892.886385:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575892.886387:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575892.886389:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788575892.886407:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575892.886409:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575892.886411:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575892.886413:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575892.886414:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575892.886417:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575892.886419:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575892.886420:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575892.886421:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575892.886423:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886428:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575892.886430:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886433:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575892.886434:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575892.886436:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575892.886439:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788575892.886441:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575892.886442:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307496487
00000100:00000001:3.0:1788575892.886444:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307497401
00000100:00000001:3.0:1788575892.886445:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307496487
00000100:00000001:3.0:1788575892.886447:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307497401
00000100:00000001:3.0:1788575892.886448:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575892.886450:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575892.886451:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575892.886455:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575892/real 1788575892]  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575892.886462:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575892.886464:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575892.886465:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575892.886467:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575892.886469:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575892.886470:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575892.886473:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575892.886480:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886486:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575892.886488:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886490:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575892.886491:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575892.886494:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886500:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575892.886504:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886509:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575892.886512:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886513:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575892.886514:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886516:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575892.886518:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575892.886521:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575892.886524:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575892.886526:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575892.886529:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575892.886537:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788575892.886541:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575892.886544:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886550:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788575892.886551:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575892.886553:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575892.886555:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575892.886557:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575892.886559:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575892.886562:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575892.886564:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575892.886565:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575892.886566:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575892.886570:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575892.886572:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788575892.886575:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788575892.886577:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788575892.886581:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788575892.886583:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393243776, portal 4
00000100:00000001:3.0:1788575892.886585:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575892.886586:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:3.0:1788575892.886592:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886599:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575892.886601:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788575892.886603:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788575892.886606:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393243776, offset 0
00000400:00000200:3.0:1788575892.886609:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575892.886613:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575892.886616:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575892.886619:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788575892.886621:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788575892.886623:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788575892.886625:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788575892.886628:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331160:331160:256:4294967295] 192.168.202.110@tcp LPNI seq info [331117:331117:8:4294967295]
00000400:00000200:3.0:1788575892.886633:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575892.886638:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788575892.886643:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575892.886646:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788575892.886650:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788575892.886653:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788575892.886656:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788575892.886660:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788575892.886663:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788575892.886665:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788575892.886668:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788575892.886671:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788575892.886673:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788575892.886674:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788575892.886675:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788575892.886678:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886683:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575892.886685:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886704:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575892.886705:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788575892.886708:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575892.886710:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788575892.886713:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575892.886714:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307496633
00000100:00000001:3.0:1788575892.886716:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307497401
00000100:00000001:3.0:1788575892.886717:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307496633
00000100:00000001:3.0:1788575892.886718:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307497401
00000100:00000001:3.0:1788575892.886720:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575892.886721:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575892.886722:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788575892.886725:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575892/real 1788575892]  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788575892.886729:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788575892.886730:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788575892.886731:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788575892.886732:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788575892.886733:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788575892.886734:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788575892.886736:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788575892.886741:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886744:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788575892.886745:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886746:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575892.886747:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788575892.886749:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886753:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788575892.886755:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886759:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788575892.886760:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886761:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788575892.886762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886763:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788575892.886764:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788575892.886765:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788575892.886768:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788575892.886770:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788575892.886773:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575892.886780:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393243776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788575892.886783:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886784:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575892.886787:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393243648/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886793:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575892.886795:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575892.886796:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788575892.886799:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575892.886800:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575892.886802:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788575892.886804:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575892.886805:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788575892.886808:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575892.886809:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575892.886810:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:3.0:1788575892.886813:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575892.886814:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886815:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788575892.886819:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393243776/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575892.886823:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788575892.886824:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788575892.886825:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788575892.886826:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788575892.886827:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788575892.886828:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788575892.886829:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788575892.886830:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:3.0:1788575892.886831:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788575892.886831:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788575892.886832:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:3.0:1788575892.886833:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788575892.886834:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886835:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788575892.886837:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788575892.886838:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575892.886839:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575892.886840:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575893.269556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575893.269565:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575893.269572:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575893.269573:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575893.269577:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575893.269579:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575893.269731:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575893.269737:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575893.269742:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575893.397561:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575893.397571:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575893.397574:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575893.397581:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575893.397586:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575893.397591:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575893.397595:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575893.397597:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575893.397601:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575893.845532:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307497401, at 4307497408
00000100:00000040:1.0:1788575893.845532:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307497401, at 4307497408
00000100:00000010:1.0:1788575893.845552:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000010:0.0:1788575893.845554:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00010000:00000001:3.0:1788575894.293515:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575894.293522:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575894.293528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575894.293530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575894.293533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575894.293536:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575894.293611:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575894.293616:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575894.293619:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575894.421495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575894.421502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575894.421505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575894.421511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575894.421515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575894.421519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575894.421521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575894.421524:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575894.421527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575895.317529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575895.317539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575895.317546:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575895.317547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575895.317551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575895.317552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575895.317633:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575895.317672:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575895.317677:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575895.445575:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575895.445587:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575895.445591:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575895.445599:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575895.445604:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575895.445610:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575895.445613:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575895.445617:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575895.445621:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575896.341544:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575896.341550:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575896.341554:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575896.341557:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575896.341560:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575896.341562:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575896.341565:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575896.341567:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575896.341568:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575896.341570:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575896.341574:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575896.341575:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575896.341577:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575896.341578:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575896.341580:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575896.341581:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575896.341583:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575896.341585:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575896.341680:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575896.341686:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575896.341689:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575896.472772:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575896.472782:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575896.472785:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575896.472790:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575896.472795:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575896.472800:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575896.472802:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575896.472805:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575896.472809:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575897.365525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575897.365533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575897.365538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575897.365539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575897.365543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575897.365544:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575897.365626:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575897.365630:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575897.365634:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575897.493508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575897.493515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575897.493517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575897.493522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575897.493524:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575897.493527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575897.493529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575897.493530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575897.493532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575898.005548:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575898.005558:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575898.005560:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575898.005565:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575898.005570:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575898.005577:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12829
00000400:00000200:3.0:1788575898.005602:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575898.005612:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575898.005615:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575898.005618:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575898.005621:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575898.005623:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575898.005625:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575898.005629:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575898.005631:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575898.005634:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575898.005637:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575898.005640:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575898.005642:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575898.005645:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575898.005647:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575898.005649:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575898.005651:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575898.005654:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575898.005657:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575898.005660:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575898.005662:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575898.005666:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575898.005669:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575898.005671:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575898.005673:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575898.005682:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000020:00000040:3.0:1788575898.005685:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575898.005689:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575898.005690:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575898.005693:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575898.005696:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575898.005699:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575898.005707:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788575898.005713:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575898.005722:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575898.005731:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575898.005735:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575898.005740:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575898.005763:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575898.005767:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575898.005771:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575898.005772:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575898.005773:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575898.005776:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575898.005780:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12829
00000400:00000200:3.0:1788575898.005786:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575898.005790:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575898.005793:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575898.005797:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575898.005797:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575898.005804:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575898.005806:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575898.005807:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575898.005812:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575898.005814:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575898.005819:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575898.005820:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575898.005824:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575898.005828:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575898.005828:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575898.005835:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575898.005843:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575898.005856:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575898.005858:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575898.005863:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575898.005864:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575898.005869:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1788575898.005870:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575898.005873:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575898.005876:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575898.005878:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575898.005887:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00100000:2.0:1788575898.005888:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393243904:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575898.005901:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1788575898.005905:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575898.005909:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1788575898.005911:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575898.005912:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575898.005920:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575898.005926:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1788575898.005927:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575898.005933:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575898.005935:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575898.005940:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1788575898.005941:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
00000008:00000001:3.0:1788575898.005944:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575898.005947:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575898.005949:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575898.005956:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788575898.005966:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:2.0:1788575898.005968:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575898.005978:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1788575898.005979:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
02000000:00000001:3.0:1788575898.005981:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575898.005985:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575898.005987:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575898.005989:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575898.005990:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
02000000:00000010:3.0:1788575898.005995:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000200:2.0:1788575898.006001:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393243904, portal 25
00000100:00000001:3.0:1788575898.006002:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.006009:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575898.006013:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00100000:3.0:1788575898.006014:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788575898.006031:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006053:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575898.006056:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575898.006059:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575898.006063:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393243904, offset 0
00000100:00000001:3.0:1788575898.006075:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575898.006079:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575898.006082:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575898.006090:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575898.006100:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575898.006111:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575898.006116:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:3.0:1788575898.006118:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575898.006125:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575898.006129:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1788575898.006130:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1788575898.006135:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788575898.006139:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575898.006142:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575898.006146:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575898.006151:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:3.0:1788575898.006153:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12829
00000400:00000200:2.0:1788575898.006158:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331161:331161:256:4294967295] 192.168.202.110@tcp LPNI seq info [331118:331118:8:4294967295]
00000400:00000200:2.0:1788575898.006163:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575898.006171:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575898.006181:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1788575898.006186:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:2.0:1788575898.006191:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575898.006191:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000010:2.0:1788575898.006196:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000100:00000040:3.0:1788575898.006200:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575898.006207:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1788575898.006208:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575898.006214:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:2.0:1788575898.006217:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:3.0:1788575898.006217:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575898.006222:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575898.006225:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575898.006228:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575898.006234:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:2.1:1788575898.006235:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788575898.006238:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.1:1788575898.006242:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:3.0:1788575898.006244:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000100:2.0:1788575898.006253:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575898.006256:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000010:2.0:1788575898.006258:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000100:00000001:3.0:1788575898.006265:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575898.006269:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575898.006271:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1788575898.006273:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575898.006280:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788575898.006287:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1788575898.006287:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575898.006289:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575898.006296:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575898.006300:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575898.006304:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
02000000:00000001:3.0:1788575898.006304:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575898.006310:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:3.0:1788575898.006311:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575898.006316:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575898.006319:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000008:00000001:3.0:1788575898.006320:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575898.006324:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575898.006328:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1788575898.006331:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000020:00000040:3.0:1788575898.006340:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575898.006348:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575898.006352:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.006355:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1788575898.006356:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575898.006361:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575898.006365:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575898.006370:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575898.006373:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575898.006379:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788575898.006384:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575898.006420:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575898.006421:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575898.006425:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006436:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575898.006438:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.006443:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:3.0:1788575898.006443:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575898.006450:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575898.006457:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12839)
00000100:00000001:2.0:1788575898.006458:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575898.006470:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575898.006474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307502522
00000100:00000010:2.0:1788575898.006480:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c840.
00000100:00000001:2.0:1788575898.006487:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575898.006493:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307502522
00000100:00000010:2.0:1788575898.006495:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c000.
00000100:00000001:2.0:1788575898.006497:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575898.006500:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575898.006502:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.006504:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575898.006509:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575898/real 1788575898]  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575898.006517:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575898.006519:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575898.006521:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575898.006523:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575898.006524:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575898.006526:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575898.006528:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575898.006538:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006544:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575898.006545:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.006546:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575898.006548:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575898.006550:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006556:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575898.006561:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006568:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575898.006571:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.006572:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575898.006574:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.006576:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575898.006579:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575898.006595:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575898.006600:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575898.006602:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575898.006607:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575898.006615:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393243904:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575898.006620:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.006622:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575898.006624:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393243904/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006635:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575898.006638:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575898.006642:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575898.006645:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575898.006648:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575898.006650:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575898.006652:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575898.006655:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575898.006659:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575898.006661:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575898.006662:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575898.006666:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575898.006667:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.006670:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.006679:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575898.006681:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.006684:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575898.006685:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575898.006687:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575898.006691:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006698:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575898.006700:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575898.006702:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575898.006704:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575898.006707:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575898.006710:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244032:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575898.006713:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575898.006716:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575898.006718:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575898.006720:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575898.006723:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575898.006726:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575898.006730:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575898.006733:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575898.006737:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575898.006740:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393244032, portal 10
00000100:00000001:2.0:1788575898.006742:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575898.006744:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788575898.006751:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006757:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575898.006759:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575898.006762:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575898.006764:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393244032, offset 0
00000400:00000200:2.0:1788575898.006768:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575898.006772:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575898.006776:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575898.006779:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575898.006782:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575898.006784:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575898.006787:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575898.006790:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331162:331162:256:4294967295] 192.168.202.110@tcp LPNI seq info [331119:331119:8:4294967295]
00000400:00000200:2.0:1788575898.006793:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575898.006800:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575898.006804:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575898.006807:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575898.006811:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575898.006813:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575898.006818:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575898.006821:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575898.006825:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575898.006827:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575898.006829:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575898.006831:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575898.006834:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575898.006836:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575898.006839:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575898.006843:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006851:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575898.006854:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006860:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.006861:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575898.006865:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575898.006869:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575898.006873:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575898.006875:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307501607
00000100:00000001:2.0:1788575898.006878:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307502522
00000100:00000001:2.0:1788575898.006880:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307501607
00000100:00000001:2.0:1788575898.006882:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307502522
00000100:00000001:2.0:1788575898.006884:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575898.006886:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.006888:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575898.006891:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575898/real 1788575898]  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575898.006900:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575898.006901:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575898.006903:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575898.006905:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575898.006907:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575898.006909:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575898.006913:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575898.006921:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006933:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575898.006934:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.006936:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575898.006937:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575898.006941:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006947:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575898.006952:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.006959:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575898.006961:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.006963:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575898.006964:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.006966:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575898.006969:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575898.006972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575898.006975:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575898.006977:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575898.006981:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575898.006989:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244032:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575898.006993:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575898.006997:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.007003:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575898.007004:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575898.007007:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575898.007010:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575898.007012:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575898.007015:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244160:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575898.007018:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575898.007020:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575898.007022:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575898.007024:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575898.007029:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575898.007031:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575898.007035:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575898.007038:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575898.007042:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575898.007044:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393244160, portal 4
00000100:00000001:2.0:1788575898.007047:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575898.007048:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575898.007054:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.007061:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575898.007063:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575898.007066:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575898.007068:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393244160, offset 0
00000400:00000200:2.0:1788575898.007073:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575898.007076:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575898.007080:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575898.007082:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575898.007085:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575898.007087:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575898.007090:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575898.007093:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331163:331163:256:4294967295] 192.168.202.110@tcp LPNI seq info [331120:331120:8:4294967295]
00000400:00000200:2.0:1788575898.007098:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575898.007104:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575898.007109:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575898.007112:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575898.007116:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575898.007119:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575898.007123:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575898.007126:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575898.007129:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575898.007131:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575898.007135:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575898.007138:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575898.007141:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575898.007143:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575898.007145:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575898.007148:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575923 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.007156:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575898.007159:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575923 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.007165:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.007167:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575898.007169:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575898.007173:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575898.007176:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575898.007178:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307501753
00000100:00000001:2.0:1788575898.007181:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307502522
00000100:00000001:2.0:1788575898.007183:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307501753
00000100:00000001:2.0:1788575898.007185:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307502522
00000100:00000001:2.0:1788575898.007187:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575898.007189:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.007191:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575898.007195:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575898/real 1788575898]  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575898.007202:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575898.007204:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575898.007205:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575898.007207:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575898.007209:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575898.007211:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575898.007214:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575898.007222:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.007228:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575898.007230:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.007232:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575898.007233:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575898.007237:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.007244:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575898.007247:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.007255:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575898.007257:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.007258:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575898.007260:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.007261:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575898.007263:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575898.007265:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575898.007268:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575898.007271:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575898.007275:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575898.007283:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244160:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575898.007287:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.007289:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575898.007292:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393244032/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.007298:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575898.007299:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575898.007301:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575898.007304:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575898.007306:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575898.007307:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575898.007309:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575898.007311:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575898.007313:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575898.007315:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575898.007316:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575898.007319:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575898.007320:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.007322:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575898.007325:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393244160/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575923 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575898.007331:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575898.007333:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575898.007334:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575898.007336:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575898.007337:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575898.007339:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575898.007340:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575898.007342:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575898.007344:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575898.007346:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575898.007347:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575898.007350:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575898.007351:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.007353:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575898.007357:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575898.007358:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575898.007360:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575898.007361:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575898.389513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575898.389520:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575898.389526:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575898.389527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575898.389530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575898.389532:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575898.389836:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575898.389840:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575898.389843:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575898.518901:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575898.518907:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575898.518909:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575898.518915:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575898.518918:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575898.518921:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575898.518923:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575898.518925:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575898.518927:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575898.965512:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307502522, at 4307502528
00000100:00000010:1.0:1788575898.965526:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00000100:00000040:0.0:1788575898.965605:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307502522, at 4307502528
00000100:00000010:0.0:1788575898.965612:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00010000:00000001:3.0:1788575899.413518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575899.413526:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575899.413528:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575899.413531:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575899.413536:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575899.413538:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575899.413540:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575899.413543:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575899.413553:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575899.413584:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575899.413588:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575899.413590:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575899.413698:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575899.413702:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575899.413705:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575899.541505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575899.541513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575899.541515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575899.541520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575899.541523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575899.541527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575899.541529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575899.541532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575899.541535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575900.437633:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575900.437640:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575900.437645:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575900.437647:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575900.437651:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575900.437652:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575900.437707:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575900.437711:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575900.437713:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575900.567875:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575900.567881:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575900.567883:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575900.567888:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575900.567892:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575900.567895:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575900.567897:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575900.567900:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575900.567902:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575901.461505:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575901.461511:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575901.461528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575901.461529:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575901.461533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575901.461534:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575901.461644:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575901.461648:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575901.461651:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575901.589495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575901.589502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575901.589504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575901.589540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575901.589544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575901.589548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575901.589550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575901.589553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575901.589556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575902.485522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575902.485529:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575902.485534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575902.485536:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575902.485539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575902.485540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575902.485679:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575902.485684:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575902.485687:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575902.613534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575902.613541:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575902.613544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575902.613549:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575902.613553:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575902.613557:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575902.613560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575902.613562:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575902.613564:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575903.125538:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575903.125545:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575903.125547:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575903.125550:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575903.125554:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575903.125560:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12834
00000400:00000200:3.0:1788575903.125567:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575903.125572:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575903.125574:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575903.125576:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125578:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575903.125580:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125582:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575903.125585:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575903.125587:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575903.125589:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575903.125591:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575903.125593:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575903.125594:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575903.125596:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125597:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575903.125598:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125599:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575903.125601:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575903.125603:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125605:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575903.125607:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575903.125610:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575903.125612:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575903.125613:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575903.125615:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575903.125623:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2300.
00000020:00000040:3.0:1788575903.125627:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575903.125630:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575903.125632:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575903.125635:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575903.125638:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575903.125642:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575903.125649:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575903.125655:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575903.125665:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575903.125675:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575903.125679:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575903.125684:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc4bce75] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575903.125714:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575903.125717:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575903.125720:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575903.125721:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575903.125723:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575903.125725:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575903.125728:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12834
00000400:00000200:3.0:1788575903.125731:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575903.125734:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575903.125736:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575903.125739:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125741:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575903.125744:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125746:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575903.125749:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575903.125750:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575903.125752:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575903.125754:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575903.125756:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788575903.125758:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575903.125758:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788575903.125763:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575903.125765:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575903.125767:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575903.125769:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575903.125772:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125776:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:0.0:1788575903.125777:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575903.125779:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575903.125785:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125791:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.125793:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1788575903.125795:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788575903.125797:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575903.125803:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575903.125803:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575903.125807:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575903.125808:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575903.125813:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575903.125818:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788575903.125820:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788575903.125821:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575903.125823:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575903.125828:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0a80.
00000020:00000040:3.0:1788575903.125832:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00100000:0.0:1788575903.125834:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393244288:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575903.125835:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575903.125839:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.125841:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575903.125842:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575903.125845:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575903.125848:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575903.125848:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575903.125853:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575903.125855:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788575903.125857:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575903.125860:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575903.125867:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:0.0:1788575903.125868:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788575903.125873:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575903.125880:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575903.125883:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575903.125884:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575903.125890:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000100:00000040:3.0:1788575903.125890:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b298c26f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575903.125901:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788575903.125904:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00080000:3.0:1788575903.125905:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575903.125911:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575903.125913:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575903.125914:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:0.0:1788575903.125916:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393244288, portal 25
00000100:00000001:3.0:1788575903.125917:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:0.0:1788575903.125921:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1788575903.125922:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12834
00000100:00000001:0.0:1788575903.125924:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000400:00000200:3.0:1788575903.125929:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575903.125934:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:0.0:1788575903.125935:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575903.125938:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575903.125945:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575903.125949:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575903.125950:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575903.125956:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575903.125956:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125961:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:0.0:1788575903.125962:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000020:00000001:3.0:1788575903.125965:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000200:0.0:1788575903.125967:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393244288, offset 0
00000020:00000040:3.0:1788575903.125968:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575903.125972:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575903.125974:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575903.125978:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788575903.125979:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:3.0:1788575903.125982:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575903.125988:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788575903.125991:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575903.125992:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575903.125997:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575903.125999:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788575903.126001:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:3.0:1788575903.126003:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575903.126008:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1788575903.126009:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788575903.126014:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575903.126016:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1788575903.126018:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575903.126022:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575903.126024:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575903.126026:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1788575903.126027:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00100000:3.0:1788575903.126029:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:0.0:1788575903.126032:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000008:00000001:3.0:1788575903.126034:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575903.126038:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575903.126039:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331164:331164:256:4294967295] 192.168.202.110@tcp LPNI seq info [331121:331121:8:4294967295]
00000100:00000010:3.0:1788575903.126044:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000400:00000200:0.0:1788575903.126048:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1788575903.126049:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575903.126053:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575903.126055:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575903.126059:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575903.126062:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1788575903.126063:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1788575903.126066:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788575903.126072:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000010:3.0:1788575903.126072:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575903.126078:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788575903.126082:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000100:00100000:3.0:1788575903.126086:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1788575903.126093:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575903.126099:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:0.0:1788575903.126100:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:3.0:1788575903.126103:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788575903.126108:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575903.126110:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1788575903.126113:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1788575903.126122:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575903.126126:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788575903.126128:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000100:00000040:3.0:1788575903.126131:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575903.126136:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575903.126141:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12844)
00000400:00000200:0.0:1788575903.126142:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575903.126149:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575903.126155:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788575903.126157:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788575903.126162:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575903.126166:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126176:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575903.126180:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126187:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.126188:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575903.126193:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575903.126197:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575903.126201:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307507641
00000100:00000010:0.0:1788575903.126207:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788575903.126214:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788575903.126219:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307507641
00000100:00000010:0.0:1788575903.126222:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7c00.
00000100:00000001:0.0:1788575903.126225:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788575903.126232:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575903.126235:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.126238:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575903.126242:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575903/real 1788575903]  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575903.126251:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575903.126253:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575903.126255:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788575903.126258:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788575903.126260:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788575903.126262:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575903.126265:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575903.126273:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126279:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575903.126281:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126283:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575903.126285:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575903.126289:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126296:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575903.126301:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126308:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575903.126311:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126312:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575903.126314:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126316:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575903.126318:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575903.126321:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575903.126325:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575903.126327:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575903.126352:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575903.126361:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393244288:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575903.126367:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126369:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575903.126372:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393244288/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126378:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575903.126380:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575903.126383:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788575903.126386:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575903.126387:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575903.126389:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788575903.126419:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575903.126439:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788575903.126444:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575903.126445:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575903.126447:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:0.0:1788575903.126450:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575903.126452:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126454:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126461:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575903.126463:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.126465:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575903.126466:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575903.126467:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575903.126471:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126488:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575903.126490:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575903.126492:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575903.126494:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575903.126496:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575903.126499:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244416:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575903.126502:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575903.126504:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575903.126506:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575903.126507:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575903.126511:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788575903.126513:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575903.126516:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575903.126519:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788575903.126523:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788575903.126526:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393244416, portal 10
00000100:00000001:0.0:1788575903.126528:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575903.126529:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:0.0:1788575903.126535:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126541:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575903.126544:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575903.126546:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788575903.126548:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393244416, offset 0
00000400:00000200:0.0:1788575903.126552:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575903.126555:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575903.126560:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575903.126563:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575903.126565:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575903.126568:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575903.126570:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575903.126573:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331165:331165:256:4294967295] 192.168.202.110@tcp LPNI seq info [331122:331122:8:4294967295]
00000400:00000200:0.0:1788575903.126576:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575903.126583:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575903.126588:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575903.126591:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788575903.126594:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575903.126597:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788575903.126601:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575903.126604:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1788575903.126607:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575903.126609:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788575903.126612:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575903.126615:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575903.126618:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788575903.126620:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788575903.126622:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575903.126626:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126633:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575903.126635:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126642:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.126643:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575903.126646:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575903.126650:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788575903.126653:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575903.126654:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307506727
00000100:00000001:0.0:1788575903.126657:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307507641
00000100:00000001:0.0:1788575903.126659:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307506727
00000100:00000001:0.0:1788575903.126661:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307507641
00000100:00000001:0.0:1788575903.126663:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575903.126665:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.126667:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575903.126671:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575903/real 1788575903]  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575903.126679:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575903.126680:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575903.126682:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788575903.126684:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788575903.126686:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788575903.126687:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575903.126691:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575903.126698:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126704:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575903.126705:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126707:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575903.126709:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575903.126712:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126718:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575903.126722:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126729:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575903.126731:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126733:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575903.126734:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126736:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575903.126738:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575903.126740:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575903.126743:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575903.126745:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575903.126750:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575903.126757:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244416:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575903.126761:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575903.126764:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126770:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575903.126771:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575903.126773:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575903.126775:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575903.126777:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575903.126780:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575903.126783:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575903.126785:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575903.126807:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575903.126809:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575903.126812:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788575903.126814:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575903.126818:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575903.126820:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788575903.126823:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788575903.126826:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393244544, portal 4
00000100:00000001:0.0:1788575903.126828:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575903.126829:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:0.0:1788575903.126835:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126841:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575903.126843:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575903.126845:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788575903.126847:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393244544, offset 0
00000400:00000200:0.0:1788575903.126852:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575903.126856:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575903.126860:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575903.126862:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575903.126864:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575903.126866:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575903.126869:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575903.126872:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331166:331166:256:4294967295] 192.168.202.110@tcp LPNI seq info [331123:331123:8:4294967295]
00000400:00000200:0.0:1788575903.126876:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575903.126881:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575903.126885:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575903.126888:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788575903.126892:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575903.126894:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788575903.126897:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575903.126900:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1788575903.126903:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575903.126905:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788575903.126908:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575903.126910:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575903.126913:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:0.0:1788575903.126915:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:0.0:1788575903.126917:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575903.126920:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575928 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126928:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575903.126931:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575928 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126937:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.126938:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575903.126941:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575903.126944:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788575903.126948:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575903.126950:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307506873
00000100:00000001:0.0:1788575903.126952:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307507641
00000100:00000001:0.0:1788575903.126954:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307506873
00000100:00000001:0.0:1788575903.126956:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307507641
00000100:00000001:0.0:1788575903.126958:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575903.126960:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.126961:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575903.126964:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575903/real 1788575903]  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575903.126971:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575903.126972:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575903.126974:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788575903.126976:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788575903.126977:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788575903.126979:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575903.126982:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575903.126990:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.126996:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575903.126997:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.126998:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575903.127000:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575903.127003:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.127009:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575903.127012:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.127018:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575903.127021:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.127022:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575903.127023:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.127025:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575903.127027:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575903.127029:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575903.127032:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575903.127034:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575903.127037:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575903.127045:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575903.127048:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.127050:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575903.127053:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393244416/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.127058:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575903.127060:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575903.127062:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788575903.127064:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575903.127066:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575903.127067:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788575903.127069:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575903.127070:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788575903.127073:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575903.127074:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575903.127076:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:0.0:1788575903.127078:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575903.127079:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.127081:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575903.127084:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393244544/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575928 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575903.127095:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575903.127096:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575903.127097:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788575903.127099:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575903.127100:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575903.127101:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788575903.127103:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575903.127104:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:0.0:1788575903.127106:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575903.127107:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575903.127108:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:0.0:1788575903.127110:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575903.127111:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.127112:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575903.127115:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575903.127116:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575903.127118:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575903.127119:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575903.129764:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1788575903.129772:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1788575903.129777:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575903.130087:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575903.130100:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575903.130104:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575903.130110:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575903.130115:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575903.509513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575903.509520:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575903.509526:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575903.509527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575903.509530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575903.509532:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575903.510888:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575903.510892:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575903.510896:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575903.637527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575903.637536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575903.637538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575903.637545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575903.637549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575903.637553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575903.637556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575903.637559:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575903.637562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575904.085521:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307507641, at 4307507648
00000100:00000010:0.0:1788575904.085537:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000040:1.0:1788575904.085692:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307507641, at 4307507648
00000100:00000010:1.0:1788575904.085697:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7c00.
00010000:00000001:3.0:1788575904.535568:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575904.535575:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575904.535580:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575904.535581:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575904.535585:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575904.535586:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575904.535734:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575904.535737:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575904.535741:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575904.661485:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575904.661492:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575904.661494:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575904.661500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575904.661503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575904.661507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575904.661510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575904.661512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575904.661515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575905.557571:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575905.557580:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575905.557586:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575905.557588:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575905.557592:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575905.557593:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575905.557718:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575905.557722:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575905.557726:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575905.685549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575905.685559:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575905.685562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575905.685569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575905.685573:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575905.685578:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575905.685581:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575905.685584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575905.685587:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575906.581555:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575906.581563:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575906.581567:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575906.581570:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575906.581573:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575906.581576:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575906.581578:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575906.581580:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575906.581581:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575906.581582:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575906.581585:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575906.581586:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575906.581587:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575906.581588:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575906.581589:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575906.581589:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575906.581590:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575906.581592:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575906.582054:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575906.582059:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575906.582062:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575906.709561:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575906.709571:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575906.709573:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575906.709579:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575906.709584:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575906.709589:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575906.709592:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575906.709595:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575906.709598:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575907.605512:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575907.605518:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575907.605524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575907.605525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575907.605529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575907.605530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575907.605653:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575907.605656:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575907.605659:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575907.733639:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575907.733645:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575907.733647:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575907.733651:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575907.733653:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575907.733657:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575907.733659:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575907.733661:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575907.733663:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575908.245862:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575908.245873:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575908.245875:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575908.245879:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575908.245883:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575908.245890:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12839
00000400:00000200:3.0:1788575908.245898:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575908.245904:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575908.245907:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575908.245910:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.245913:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575908.245915:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.245917:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575908.245922:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575908.245924:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575908.245927:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575908.245930:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575908.245932:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575908.245934:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575908.245937:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.245939:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575908.245940:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.245942:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575908.245945:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575908.245948:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575908.245951:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575908.245953:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575908.245957:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575908.245959:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575908.245961:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575908.245964:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575908.245972:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788575908.245976:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575908.245979:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575908.245981:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575908.245983:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575908.245986:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575908.245989:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575908.245996:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575908.246003:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575908.246011:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575908.246020:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575908.246024:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575908.246028:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575908.246057:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575908.246061:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575908.246064:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575908.246065:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575908.246067:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575908.246070:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575908.246072:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12839
00000400:00000200:3.0:1788575908.246075:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575908.246078:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575908.246080:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575908.246083:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246085:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575908.246086:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246088:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575908.246090:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575908.246091:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575908.246093:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575908.246095:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575908.246097:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575908.246099:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575908.246102:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246104:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575908.246106:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246108:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575908.246110:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575908.246112:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246114:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575908.246117:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575908.246122:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575908.246124:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575908.246126:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575908.246129:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575908.246132:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575908.246134:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575908.246137:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000020:00000040:3.0:1788575908.246140:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575908.246142:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575908.246143:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575908.246145:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575908.246146:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575908.246148:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575908.246152:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575908.246155:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575908.246160:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575908.246168:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575908.246170:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575908.246173:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575908.246181:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575908.246183:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575908.246186:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575908.246187:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575908.246189:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575908.246191:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575908.246193:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12839
00000400:00000200:3.0:1788575908.246197:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575908.246200:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575908.246202:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575908.246205:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246207:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575908.246208:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246210:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575908.246212:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575908.246213:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575908.246214:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575908.246216:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575908.246218:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575908.246220:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575908.246223:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246225:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575908.246227:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246229:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575908.246231:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575908.246234:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575908.246236:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575908.246238:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575908.246240:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575908.246242:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575908.246244:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575908.246246:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575908.246248:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575908.246250:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575908.246253:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575908.246255:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575908.246258:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575908.246259:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575908.246260:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575908.246261:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575908.246263:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575908.246266:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788575908.246270:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575908.246274:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575908.246280:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575908.246282:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575908.246285:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575908.246292:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575908.246294:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575908.246298:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575908.246303:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12849)
00000100:00000001:0.0:1788575908.246416:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575908.246421:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575908.246423:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575908.246430:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575908.246439:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575908.246441:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575908.246444:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575908.246447:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575908.246449:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575908.246452:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393244672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575908.246456:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575908.246460:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575908.246462:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575908.246465:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575908.246470:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788575908.246472:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575908.246477:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575908.246481:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788575908.246487:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788575908.246492:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393244672, portal 25
00000100:00000001:0.0:1788575908.246494:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575908.246496:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:0.0:1788575908.246501:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575908.246508:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575908.246510:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575908.246513:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4440.
00000100:00000200:0.0:1788575908.246515:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393244672, offset 0
00000400:00000200:0.0:1788575908.246521:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575908.246528:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575908.246534:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575908.246539:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575908.246542:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575908.246545:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575908.246548:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575908.246552:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331167:331167:256:4294967295] 192.168.202.110@tcp LPNI seq info [331124:331124:8:4294967295]
00000400:00000200:0.0:1788575908.246556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575908.246564:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575908.246571:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575908.246575:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788575908.246581:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575908.246585:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575908.246599:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575908.246603:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575908.246606:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307512762
00000100:00000010:0.0:1788575908.246611:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788575908.246617:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788575908.246622:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307512762
00000100:00000010:0.0:1788575908.246625:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7cc0.
00000100:00000001:0.0:1788575908.246629:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788575908.246633:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575908.246636:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575908.246639:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575908.246643:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575908.246649:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575908.246651:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575908.246653:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575908.246655:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575908.246657:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575908.246661:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575908.246664:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575908.246666:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575908.246667:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575908.246669:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575908.246672:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575908.246674:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575908.246678:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575908.246680:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da580.
00000400:00000010:0.0:1788575908.246684:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575908.246688:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393244800, portal 10
00000100:00000001:0.0:1788575908.246689:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575908.246691:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:0.0:1788575908.246697:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575908.246702:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575908.246704:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575908.246707:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d42a8.
00000100:00000200:0.0:1788575908.246709:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393244800, offset 0
00000400:00000200:0.0:1788575908.246713:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575908.246717:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575908.246721:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575908.246723:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575908.246725:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575908.246727:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575908.246730:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575908.246733:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331168:331168:255:4294967295] 192.168.202.110@tcp LPNI seq info [331125:331125:7:4294967295]
00000400:00000200:0.0:1788575908.246736:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575908.246742:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575908.246746:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575908.246749:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9300.
00000800:00000200:0.0:1788575908.246752:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575908.246756:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575908.246759:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575908.246762:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788575908.246765:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575908.246767:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307511847
00000100:00000001:0.0:1788575908.246769:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307512762
00000100:00000001:0.0:1788575908.246771:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307511847
00000100:00000001:0.0:1788575908.246773:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307512762
00000100:00000001:0.0:1788575908.246775:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575908.246777:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575908.246779:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575908.246783:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575908.246789:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575908.246791:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575908.246792:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575908.246794:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575908.246796:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575908.246798:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244928:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575908.246801:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575908.246803:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575908.246804:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575908.246806:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575908.246808:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575908.246810:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575908.246813:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575908.246815:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575908.246817:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575908.246820:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393244928, portal 4
00000100:00000001:0.0:1788575908.246821:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575908.246822:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:0.0:1788575908.246827:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575908.246832:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575908.246834:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575908.246836:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4770.
00000100:00000200:0.0:1788575908.246838:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393244928, offset 0
00000400:00000200:0.0:1788575908.246842:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575908.246845:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575908.246847:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575908.246850:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575908.246852:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575908.246854:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575908.246856:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575908.246859:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331169:331169:254:4294967295] 192.168.202.110@tcp LPNI seq info [331126:331126:6:4294967295]
00000400:00000200:0.0:1788575908.246862:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575908.246867:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575908.246871:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575908.246874:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788575908.246877:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575908.246880:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575908.246883:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575908.246885:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788575908.246888:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575908.246889:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307511993
00000100:00000001:0.0:1788575908.246891:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307512762
00000100:00000001:0.0:1788575908.246893:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307511993
00000100:00000001:0.0:1788575908.246895:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307512762
00000100:00000001:0.0:1788575908.246897:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575908.246899:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575908.246901:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575908.246904:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575908.246910:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575908.246913:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788575908.246915:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575908.246916:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575908.246917:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575908.246919:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575908.629721:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575908.629729:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575908.629734:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575908.629735:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575908.629739:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575908.629740:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575908.629903:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575908.629906:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575908.629908:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575908.758269:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575908.758277:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575908.758279:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575908.758285:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575908.758289:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575908.758293:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575908.758296:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575908.758298:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575908.758302:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575909.205509:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307512762, at 4307512768
00000100:00000010:1.0:1788575909.205525:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7cc0.
00000100:00000040:0.0:1788575909.205616:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307512762, at 4307512768
00000100:00000010:0.0:1788575909.205622:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00010000:00000001:3.0:1788575909.653942:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575909.653948:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575909.653950:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575909.653952:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575909.653956:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575909.653957:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575909.653959:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575909.653960:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575909.653963:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575909.653964:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575909.653966:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575909.653967:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575909.654821:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575909.654825:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575909.654828:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575909.781499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575909.781505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575909.781508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575909.781513:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575909.781517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575909.781520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575909.781523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575909.781526:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575909.781529:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575910.677485:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575910.677491:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575910.677497:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575910.677498:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575910.677502:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575910.677503:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575910.677564:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575910.677567:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575910.677571:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575910.805516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575910.805521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575910.805523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575910.805528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575910.805531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575910.805535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575910.805538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575910.805540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575910.805544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788575911.317659:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788575911.317692:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788575911.317699:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575911.317703:0:2349:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788575911.317707:0:2349:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575911.317711:0:2349:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788575911.317716:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575911.317720:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:1.0:1788575911.317725:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575911.317730:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575911.317736:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4440
00000400:00000010:1.0:1788575911.317739:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4440.
00000100:00000001:1.0:1788575911.317744:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575911.317751:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575911.317779:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575911.317784:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575911.317792:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575911.317794:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575911.317803:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575911.317806:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9300.
00000400:00000200:1.0:1788575911.317809:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575911.317812:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575911.317815:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d42a8
00000400:00000010:1.0:1788575911.317817:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d42a8.
00000100:00000001:1.0:1788575911.317819:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575911.317823:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575911.317832:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575911.317834:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575911.317841:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575911.317842:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575911.317847:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575911.317849:0:2349:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:1.0:1788575911.317852:0:2349:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575911.317855:0:2349:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575911.317858:0:2349:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4770
00000400:00000010:1.0:1788575911.317860:0:2349:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4770.
00000100:00000001:1.0:1788575911.317862:0:2349:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575911.317865:0:2349:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575933 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575911.317875:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575911.317878:0:2349:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575933 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575911.317884:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575911.317886:0:2349:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575911.318239:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575911.318243:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575911.318246:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575911.318254:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575908/real 1788575911]  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575911.318265:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575911.318267:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575911.318270:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788575911.318273:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788575911.318275:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788575911.318277:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575911.318281:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575911.318288:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318294:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575911.318296:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318298:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575911.318300:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575911.318304:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318309:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575911.318314:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318320:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575911.318323:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318324:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575911.318325:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318338:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575911.318340:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575911.318344:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575911.318350:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575911.318353:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575911.318356:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575911.318364:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393244672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575911.318369:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575911.318373:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575908/real 1788575911]  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575911.318380:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575911.318381:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575911.318383:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da580.
00000400:00000200:0.0:1788575911.318386:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575911.318387:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575911.318390:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575911.318418:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575911.318425:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318430:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575911.318432:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318433:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575911.318434:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575911.318437:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318443:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575911.318446:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318452:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575911.318455:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318456:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575911.318457:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318458:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575911.318460:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575911.318462:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575911.318465:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575911.318468:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575911.318471:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575911.318477:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575911.318480:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575911.318483:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575908/real 1788575911]  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575911.318489:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575911.318490:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575911.318492:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575911.318494:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575911.318495:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575911.318497:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575911.318500:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575911.318506:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318511:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575911.318512:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318513:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575911.318514:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575911.318517:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318523:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575911.318526:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318531:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575911.318533:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318534:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575911.318535:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318537:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575911.318538:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575911.318540:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575911.318543:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575911.318545:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575911.318547:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575911.318554:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393244928:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575911.318557:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318559:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575911.318562:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393244672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318567:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575911.318570:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575911.318574:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788575911.318578:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575911.318581:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575911.318583:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788575911.318585:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575911.318587:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:0.0:1788575911.318591:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575911.318593:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575911.318595:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:0.0:1788575911.318599:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575911.318600:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318601:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575911.318605:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393244800/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318611:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575911.318613:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575911.318614:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575911.318617:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575911.318618:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575911.318619:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788575911.318621:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575911.318622:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:0.0:1788575911.318625:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575911.318626:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575911.318627:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:0.0:1788575911.318629:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575911.318630:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318632:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575911.318635:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393244928/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575933 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575911.318641:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575911.318642:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575911.318644:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575911.318646:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575911.318648:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575911.318649:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788575911.318650:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575911.318652:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:0.0:1788575911.318654:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575911.318655:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575911.318656:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:0.0:1788575911.318658:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575911.318659:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318662:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575911.318670:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575911.318672:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575911.318674:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575911.318676:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575911.703577:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575911.703587:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575911.703594:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575911.703595:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575911.703599:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575911.703601:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575911.703786:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575911.703791:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575911.703794:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575911.829557:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575911.829568:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575911.829570:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575911.829578:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575911.829582:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575911.829586:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575911.829589:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575911.829592:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575911.829596:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575912.725580:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575912.725586:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575912.725591:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575912.725593:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575912.725596:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575912.725597:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575912.726710:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575912.726715:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575912.726719:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575912.853678:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575912.853685:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575912.853687:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575912.853693:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575912.853696:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575912.853700:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575912.853703:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575912.853705:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575912.853708:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575913.365495:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575913.365506:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575913.365508:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575913.365512:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575913.365516:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575913.365522:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12844
00000400:00000200:3.0:1788575913.365531:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575913.365537:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575913.365539:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575913.365542:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365545:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575913.365547:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365550:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575913.365554:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575913.365556:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575913.365558:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575913.365561:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575913.365563:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575913.365565:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575913.365569:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365571:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575913.365572:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365574:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575913.365577:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575913.365580:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365583:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575913.365586:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575913.365589:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575913.365591:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575913.365594:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575913.365596:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575913.365602:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575913.365606:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575913.365609:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575913.365611:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575913.365613:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575913.365615:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575913.365619:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575913.365625:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
00000100:00000001:3.0:1788575913.365632:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575913.365640:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575913.365649:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575913.365653:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575913.365657:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575913.365680:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575913.365683:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575913.365686:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575913.365687:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575913.365689:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575913.365692:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575913.365695:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12844
00000400:00000200:3.0:1788575913.365698:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575913.365701:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575913.365703:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575913.365705:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365708:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575913.365710:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365712:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575913.365715:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575913.365716:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575913.365718:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575913.365720:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575913.365722:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575913.365724:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788575913.365725:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575913.365729:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575913.365732:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575913.365733:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788575913.365736:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575913.365737:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365741:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575913.365745:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:0.0:1788575913.365747:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575913.365750:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365756:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575913.365760:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788575913.365764:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575913.365767:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575913.365768:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575913.365771:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575913.365772:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575913.365775:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575913.365776:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1788575913.365781:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575913.365783:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00100000:0.0:1788575913.365786:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393245056:192.168.202.110@tcp:250:kworker.0
00000008:00000001:3.0:1788575913.365789:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.365793:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575913.365795:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575913.365799:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:3.0:1788575913.365801:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
02000000:00000001:0.0:1788575913.365803:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575913.365805:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575913.365808:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1788575913.365812:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575913.365815:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575913.365816:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:3.0:1788575913.365818:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575913.365820:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575913.365822:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575913.365826:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575913.365830:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1788575913.365834:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa000.
00000400:00000010:0.0:1788575913.365836:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000100:00000001:3.0:1788575913.365841:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788575913.365844:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00100000:3.0:1788575913.365849:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1788575913.365853:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393245056, portal 25
00000100:00000001:0.0:1788575913.365858:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575913.365861:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000001:3.0:1788575913.365863:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575913.365867:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575913.365871:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575913.365876:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.365888:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575913.365895:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575913.365896:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575913.365902:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00080000:3.0:1788575913.365902:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:0.0:1788575913.365907:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393245056, offset 0
00000100:00080000:3.0:1788575913.365908:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575913.365912:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575913.365914:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1788575913.365917:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575913.365918:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575913.365924:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12844
00000400:00000200:0.0:1788575913.365928:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575913.365933:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575913.365939:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788575913.365941:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575913.365946:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1788575913.365948:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1788575913.365952:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788575913.365956:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575913.365956:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575913.365962:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788575913.365964:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575913.365968:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788575913.365970:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:3.0:1788575913.365974:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1788575913.365979:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331170:331170:256:4294967295] 192.168.202.110@tcp LPNI seq info [331127:331127:8:4294967295]
00000020:00000040:3.0:1788575913.365979:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575913.365983:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575913.365987:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:0.0:1788575913.365989:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1788575913.365993:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575913.365998:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788575913.366006:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1788575913.366007:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575913.366013:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1788575913.366018:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575913.366018:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575913.366023:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000010:0.0:1788575913.366027:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000020:00000040:3.0:1788575913.366034:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:0.0:1788575913.366036:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1788575913.366041:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:0.0:1788575913.366042:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:3.0:1788575913.366047:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575913.366053:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1788575913.366053:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575913.366058:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
02000000:00000001:3.0:1788575913.366059:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575913.366064:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1788575913.366066:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575913.366067:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000010:0.0:1788575913.366070:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000008:00100000:3.0:1788575913.366071:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575913.366076:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575913.366077:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1788575913.366079:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575913.366085:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000010:3.0:1788575913.366086:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788575913.366090:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:0.0:1788575913.366092:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
02000000:00000001:3.0:1788575913.366093:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:0.0:1788575913.366096:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
02000000:00000001:3.0:1788575913.366096:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575913.366101:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1788575913.366102:0:2361:0:(events.c:35:request_out_callback()) Process entered
02000000:00000001:3.0:1788575913.366103:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575913.366107:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788575913.366110:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788575913.366116:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788575913.366122:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.366125:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00100000:3.0:1788575913.366131:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1788575913.366132:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575913.366156:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575913.366157:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575913.366161:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.366162:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575913.366170:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575913.366170:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788575913.366178:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575913.366183:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307517881
00000100:00000001:3.0:1788575913.366188:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575913.366191:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000040:3.0:1788575913.366193:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575913.366198:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:0.0:1788575913.366199:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:3.0:1788575913.366204:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12854)
00000100:00000001:0.0:1788575913.366206:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307517881
00000100:00000010:0.0:1788575913.366211:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7de0.
00000100:00000001:0.0:1788575913.366215:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788575913.366219:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575913.366221:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.366223:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575913.366228:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575913/real 1788575913]  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575913.366236:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575913.366238:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575913.366240:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575913.366242:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575913.366244:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575913.366246:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575913.366249:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575913.366256:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.366262:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575913.366264:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.366266:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575913.366269:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575913.366272:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.366278:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575913.366282:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.366289:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575913.366292:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.366293:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575913.366294:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.366297:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575913.366299:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575913.366302:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575913.366305:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575913.366337:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575913.366342:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575913.366350:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393245056:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575913.366354:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.366356:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575913.366358:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393245056/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.366364:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575913.366366:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575913.366369:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575913.366372:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575913.366374:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575913.366376:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788575913.366378:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575913.366381:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:0.0:1788575913.366384:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575913.366386:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575913.366387:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:0.0:1788575913.366422:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575913.366423:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.366426:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.366435:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575913.366437:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.366439:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575913.366440:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575913.366442:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575913.366445:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.366451:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575913.366468:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575913.366470:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575913.366472:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575913.366474:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575913.366476:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245184:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575913.366479:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575913.366482:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575913.366483:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575913.366485:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575913.366488:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575913.366490:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575913.366493:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575913.366496:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575913.366499:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575913.366502:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393245184, portal 10
00000100:00000001:0.0:1788575913.366504:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575913.366505:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:0.0:1788575913.366511:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.366516:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575913.366518:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575913.366520:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575913.366522:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393245184, offset 0
00000400:00000200:0.0:1788575913.366525:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575913.366529:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575913.366534:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575913.366537:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575913.366539:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575913.366541:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575913.366544:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575913.366547:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331171:331171:256:4294967295] 192.168.202.110@tcp LPNI seq info [331128:331128:8:4294967295]
00000400:00000200:0.0:1788575913.366551:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575913.366556:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575913.366561:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575913.366564:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575913.366567:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575913.366570:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788575913.366574:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575913.366577:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788575913.366580:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575913.366581:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575913.366584:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575913.366587:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575913.366589:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575913.366591:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575913.366593:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575913.366596:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.366603:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575913.366606:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.366612:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.366613:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575913.366616:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575913.367669:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788575913.367680:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575913.367683:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307516968
00000100:00000001:0.0:1788575913.367688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307517881
00000100:00000001:0.0:1788575913.367690:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307516968
00000100:00000001:0.0:1788575913.367692:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307517881
00000100:00000001:0.0:1788575913.367695:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575913.367698:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.367700:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575913.367707:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575913/real 1788575913]  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575913.367717:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575913.367719:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575913.367721:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575913.367725:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575913.367727:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575913.367730:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575913.367733:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575913.367741:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.367748:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575913.367749:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.367751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575913.367753:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575913.367756:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.367763:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575913.367768:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.367774:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575913.367776:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.367778:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575913.367779:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.367781:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575913.367783:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575913.367786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575913.367790:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575913.367792:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575913.367795:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575913.367802:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245184:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575913.367806:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575913.367809:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.367815:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575913.367817:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575913.367819:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575913.367822:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575913.367824:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575913.367827:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245312:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575913.367830:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575913.367833:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575913.367834:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575913.367836:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575913.367842:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575913.367844:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575913.367847:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575913.367851:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575913.367854:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575913.367857:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393245312, portal 4
00000100:00000001:0.0:1788575913.367859:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575913.367860:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:0.0:1788575913.367867:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.367872:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575913.367875:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575913.367877:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575913.367879:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393245312, offset 0
00000400:00000200:0.0:1788575913.367884:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575913.367889:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575913.367895:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575913.367898:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575913.367900:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575913.367903:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575913.367906:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575913.367909:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331172:331172:256:4294967295] 192.168.202.110@tcp LPNI seq info [331129:331129:8:4294967295]
00000400:00000200:0.0:1788575913.367913:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575913.367920:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575913.367925:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575913.367929:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575913.367934:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575913.367937:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788575913.367941:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575913.367944:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788575913.367948:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575913.367950:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575913.367955:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575913.367958:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575913.367962:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575913.367964:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575913.367967:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575913.367970:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575938 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.367978:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575913.367981:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575938 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.367988:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.367989:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575913.367993:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575913.367997:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788575913.368001:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575913.368003:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307517114
00000100:00000001:0.0:1788575913.368005:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307517881
00000100:00000001:0.0:1788575913.368007:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307517114
00000100:00000001:0.0:1788575913.368010:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307517881
00000100:00000001:0.0:1788575913.368012:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575913.368013:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.368015:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575913.368019:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575913/real 1788575913]  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575913.368027:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575913.368028:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575913.368030:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575913.368032:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575913.368034:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575913.368036:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575913.368039:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575913.368046:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.368054:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575913.368055:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.368056:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575913.368057:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575913.368061:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.368066:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575913.368070:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.368075:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575913.368077:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.368079:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575913.368080:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.368081:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575913.368083:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575913.368085:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575913.368088:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575913.368090:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575913.368093:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575913.368100:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245312:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575913.368105:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.368106:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575913.368109:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393245184/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.368116:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575913.368118:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575913.368121:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575913.368124:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575913.368126:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575913.368128:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788575913.368130:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575913.368132:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:0.0:1788575913.368136:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575913.368137:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575913.368139:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:0.0:1788575913.368141:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575913.368143:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.368144:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575913.368148:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393245312/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575938 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575913.368154:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575913.368155:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575913.368156:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575913.368158:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575913.368160:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575913.368161:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788575913.368163:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575913.368164:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:0.0:1788575913.368166:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575913.368168:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575913.368169:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:0.0:1788575913.368172:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575913.368173:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.368175:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575913.368182:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575913.368184:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575913.368186:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575913.368188:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575913.749534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575913.749541:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575913.749554:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575913.749555:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575913.749559:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575913.749560:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575913.749720:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575913.749723:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575913.749726:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575913.877496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575913.877502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575913.877503:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575913.877508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575913.877510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575913.877513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575913.877516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575913.877518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575913.877519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575914.325506:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307517881, at 4307517888
00000100:00000040:1.0:1788575914.325506:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307517881, at 4307517888
00000100:00000010:0.0:1788575914.325521:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:1.0:1788575914.325527:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7de0.
00010000:00000001:3.0:1788575914.773588:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575914.773597:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575914.773605:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575914.773606:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575914.773610:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575914.773611:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575914.773727:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575914.773731:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575914.773734:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575914.901560:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575914.901571:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575914.901574:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575914.901580:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575914.901584:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575914.901589:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575914.901592:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575914.901595:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575914.901597:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575915.797486:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575915.797492:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575915.797498:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575915.797501:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575915.797503:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575915.797506:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575915.797509:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575915.797511:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575915.797513:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575915.797515:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575915.797518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575915.797520:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575915.797521:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575915.797523:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575915.797524:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575915.797525:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575915.797527:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575915.797529:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575915.797607:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575915.797610:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575915.797613:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575915.925508:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575915.925520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575915.925521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575915.925526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575915.925529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575915.925531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575915.925533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575915.925535:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575915.925537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575916.821520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575916.821527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575916.821533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575916.821534:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575916.821538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575916.821539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575916.821604:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575916.821608:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575916.821612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575916.949507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575916.949515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575916.949517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575916.949523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575916.949527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575916.949531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575916.949534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575916.949536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575916.949540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575917.845549:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575917.845556:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575917.845560:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575917.845561:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575917.845564:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575917.845565:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575917.845622:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575917.845625:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575917.845629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575917.973574:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575917.973586:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575917.973589:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575917.973597:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575917.973602:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575917.973608:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575917.973611:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575917.973619:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575917.973623:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575918.485553:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575918.485567:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575918.485570:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575918.485582:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575918.485588:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575918.485596:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12849
00000400:00000200:3.0:1788575918.485607:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575918.485614:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575918.485618:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575918.485621:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.485625:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575918.485628:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.485631:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575918.485635:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575918.485641:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575918.485644:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575918.485648:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575918.485650:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575918.485653:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575918.485656:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.485659:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575918.485661:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.485663:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575918.485666:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575918.485670:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575918.485672:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575918.485675:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575918.485679:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575918.485682:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575918.485684:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575918.485687:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575918.485698:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788575918.485701:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575918.485706:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575918.485708:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575918.485712:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575918.485715:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575918.485719:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575918.485727:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575918.485735:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575918.485745:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575918.485757:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575918.485762:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575918.485768:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575918.485793:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575918.485797:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575918.485800:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575918.485802:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575918.485804:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575918.485807:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575918.485810:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12849
00000400:00000200:3.0:1788575918.485814:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575918.485817:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575918.485820:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575918.485823:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.485825:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575918.485828:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575918.485831:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575918.485831:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575918.485835:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1788575918.485837:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1788575918.485838:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:0.0:1788575918.485841:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788575918.485841:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575918.485844:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575918.485848:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575918.485850:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1788575918.485851:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575918.485859:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.485864:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575918.485868:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.485873:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575918.485877:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788575918.485881:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1788575918.485882:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575918.485885:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575918.485886:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575918.485890:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575918.485893:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575918.485898:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575918.485898:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575918.485903:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575918.485905:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575918.485907:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:0.0:1788575918.485914:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393245440:192.168.202.110@tcp:250:kworker.0
00000008:00100000:3.0:1788575918.485914:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575918.485919:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.485922:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575918.485923:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575918.485929:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
02000000:00000001:0.0:1788575918.485931:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:3.0:1788575918.485932:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575918.485936:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575918.485936:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575918.485939:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575918.485942:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575918.485943:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575918.485947:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575918.485948:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575918.485952:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000010:3.0:1788575918.485953:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
02000000:00000001:0.0:1788575918.485956:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575918.485959:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575918.485966:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1788575918.485967:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1788575918.485974:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000100:00000001:3.0:1788575918.485979:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575918.485983:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788575918.485985:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000040:3.0:1788575918.485987:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575918.485994:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393245440, portal 25
00000100:00000001:3.0:1788575918.485998:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.485999:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1788575918.486003:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1788575918.486004:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00080000:3.0:1788575918.486011:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575918.486013:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575918.486016:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:0.0:1788575918.486017:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575918.486021:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:0.0:1788575918.486027:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575918.486036:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575918.486036:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12849
00000400:00000010:0.0:1788575918.486040:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000400:00000200:3.0:1788575918.486044:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:0.0:1788575918.486045:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393245440, offset 0
00000400:00000200:3.0:1788575918.486049:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1788575918.486053:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788575918.486053:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575918.486060:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.486062:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575918.486064:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.486066:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575918.486068:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575918.486069:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575918.486071:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1788575918.486074:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1788575918.486074:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575918.486077:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575918.486081:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788575918.486082:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1788575918.486087:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575918.486090:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788575918.486091:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1788575918.486094:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788575918.486097:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575918.486098:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575918.486102:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:0.0:1788575918.486103:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575918.486109:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:3.0:1788575918.486109:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575918.486115:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575918.486118:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331173:331173:256:4294967295] 192.168.202.110@tcp LPNI seq info [331130:331130:8:4294967295]
02000000:00000001:3.0:1788575918.486119:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1788575918.486124:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575918.486129:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575918.486134:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575918.486136:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1788575918.486141:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:0.0:1788575918.486145:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00100000:3.0:1788575918.486146:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575918.486150:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788575918.486152:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000100:00000001:3.0:1788575918.486152:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575918.486156:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000800:00000200:0.0:1788575918.486161:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1788575918.486162:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575918.486168:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575918.486169:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788575918.486170:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:3.0:1788575918.486172:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575918.486174:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575918.486177:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788575918.486182:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:3.0:1788575918.486183:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575918.486189:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788575918.486189:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00100000:3.0:1788575918.486197:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:0.0:1788575918.486200:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575918.486205:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000100:00000001:3.0:1788575918.486211:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575918.486213:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575918.486218:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788575918.486220:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575918.486231:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1788575918.486232:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575918.486237:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788575918.486241:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000100:00000040:3.0:1788575918.486242:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:0.0:1788575918.486247:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000040:3.0:1788575918.486249:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12859)
00000100:00000001:0.0:1788575918.486256:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575918.486283:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486294:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575918.486299:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486313:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.486315:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575918.486321:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575918.486324:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575918.486327:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307523001
00000100:00000010:0.0:1788575918.486333:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788575918.486341:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788575918.486347:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307523001
00000100:00000010:0.0:1788575918.486350:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d76c0.
00000100:00000001:0.0:1788575918.486353:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788575918.486356:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575918.486360:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.486362:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575918.486367:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575918/real 1788575918]  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575918.486377:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575918.486379:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575918.486381:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575918.486383:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575918.486386:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575918.486388:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575918.486435:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575918.486444:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486450:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575918.486452:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.486454:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575918.486459:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575918.486471:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486478:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575918.486483:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486491:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575918.486494:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.486496:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575918.486497:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.486499:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575918.486501:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575918.486505:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575918.486509:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575918.486512:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575918.486517:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575918.486525:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393245440:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575918.486530:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.486532:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575918.486541:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393245440/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486548:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575918.486550:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575918.486553:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575918.486556:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575918.486557:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575918.486558:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788575918.486561:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575918.486564:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:0.0:1788575918.486567:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575918.486568:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575918.486570:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:0.0:1788575918.486573:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575918.486574:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.486575:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.486582:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575918.486584:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.486586:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575918.486587:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575918.486588:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575918.486591:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486595:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575918.486596:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575918.486598:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575918.486599:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575918.486601:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575918.486608:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575918.486612:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575918.486614:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575918.486616:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575918.486621:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575918.486624:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575918.486626:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575918.486629:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575918.486632:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575918.486636:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575918.486639:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393245568, portal 10
00000100:00000001:0.0:1788575918.486641:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575918.486642:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:0.0:1788575918.486649:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486655:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575918.486658:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575918.486660:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575918.486662:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393245568, offset 0
00000400:00000200:0.0:1788575918.486666:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575918.486669:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575918.486673:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575918.486675:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575918.486677:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575918.486679:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575918.486681:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575918.486684:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331174:331174:256:4294967295] 192.168.202.110@tcp LPNI seq info [331131:331131:8:4294967295]
00000400:00000200:0.0:1788575918.486686:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575918.486691:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575918.486694:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575918.486696:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575918.486699:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575918.486701:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788575918.486704:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575918.486706:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788575918.486708:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575918.486710:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575918.486712:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575918.486714:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575918.486716:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575918.486717:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575918.486719:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575918.486722:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575918.486732:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486738:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.486739:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575918.486742:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575918.486745:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788575918.486748:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575918.486749:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307522087
00000100:00000001:0.0:1788575918.486752:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307523001
00000100:00000001:0.0:1788575918.486754:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307522087
00000100:00000001:0.0:1788575918.486756:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307523001
00000100:00000001:0.0:1788575918.486758:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575918.486759:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.486761:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575918.486764:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575918/real 1788575918]  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575918.486776:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575918.486778:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575918.486779:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575918.486781:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575918.486782:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575918.486784:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575918.486786:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575918.486793:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486799:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575918.486800:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.486801:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575918.486803:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575918.486805:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486811:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575918.486814:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486820:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575918.486822:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.486823:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575918.486824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.486826:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575918.486828:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575918.486830:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575918.486832:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575918.486834:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575918.486838:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575918.486844:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575918.486848:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575918.486851:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486856:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575918.486857:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575918.486859:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575918.486861:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575918.486862:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575918.486865:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575918.486868:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575918.486870:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575918.486871:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575918.486873:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575918.486875:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575918.486877:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575918.486880:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575918.486883:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575918.486886:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575918.486888:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393245696, portal 4
00000100:00000001:0.0:1788575918.486890:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575918.486891:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:0.0:1788575918.486896:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486902:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575918.486904:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575918.486906:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575918.486908:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393245696, offset 0
00000400:00000200:0.0:1788575918.486913:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575918.486916:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575918.486920:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575918.486922:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575918.486925:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575918.486927:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575918.486929:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575918.486932:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331175:331175:256:4294967295] 192.168.202.110@tcp LPNI seq info [331132:331132:8:4294967295]
00000400:00000200:0.0:1788575918.486936:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575918.486942:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575918.486945:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575918.486948:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575918.486950:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575918.486952:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788575918.486954:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575918.486956:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788575918.486958:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575918.486960:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575918.486962:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575918.486964:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575918.486966:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575918.486967:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575918.486969:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575918.486971:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486976:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575918.486977:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.486981:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.486982:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575918.486984:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575918.486986:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788575918.486988:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575918.486990:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307522233
00000100:00000001:0.0:1788575918.486992:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307523001
00000100:00000001:0.0:1788575918.486993:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307522233
00000100:00000001:0.0:1788575918.486994:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307523001
00000100:00000001:0.0:1788575918.486995:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575918.486997:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.486998:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575918.487000:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575918/real 1788575918]  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575918.487004:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575918.487005:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575918.487006:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575918.487007:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575918.487008:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575918.487010:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575918.487012:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575918.487017:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.487020:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575918.487021:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.487022:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575918.487022:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575918.487024:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.487028:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575918.487030:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.487033:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575918.487035:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.487036:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575918.487037:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.487038:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575918.487039:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575918.487041:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575918.487043:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575918.487044:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575918.487046:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575918.487050:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575918.487052:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.487053:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575918.487055:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393245568/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.487059:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575918.487060:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575918.487061:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575918.487062:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575918.487063:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575918.487064:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788575918.487066:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575918.487067:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:0.0:1788575918.487069:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575918.487070:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575918.487070:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:0.0:1788575918.487072:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575918.487073:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.487074:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575918.487076:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393245696/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575918.487101:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575918.487102:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575918.487104:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575918.487106:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575918.487107:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575918.487108:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788575918.487110:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575918.487112:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:0.0:1788575918.487114:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575918.487115:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575918.487116:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:0.0:1788575918.487119:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575918.487120:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.487122:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575918.487124:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575918.487126:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575918.487127:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575918.487128:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575918.869529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575918.869537:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575918.869539:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575918.869543:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575918.869549:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575918.869552:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575918.869554:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575918.869557:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575918.869561:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575918.869563:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575918.869567:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575918.869568:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575918.873275:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575918.873282:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575918.873287:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575918.997555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575918.997565:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575918.997567:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575918.997574:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575918.997579:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575918.997583:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575918.997586:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575918.997589:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575918.997591:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575919.445536:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307523001, at 4307523008
00000100:00000010:0.0:1788575919.445553:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000040:1.0:1788575919.445691:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307523001, at 4307523008
00000100:00000010:1.0:1788575919.445708:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d76c0.
00010000:00000001:3.0:1788575919.893507:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575919.893512:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575919.893517:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575919.893518:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575919.893522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575919.893523:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575919.893596:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575919.893599:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575919.893603:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575920.021514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575920.021521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575920.021523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575920.021528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575920.021532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575920.021536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575920.021539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575920.021541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575920.021544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575920.917526:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575920.917532:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575920.917537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575920.917538:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575920.917542:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575920.917543:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575920.917602:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575920.917605:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575920.917608:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575921.045511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575921.045519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575921.045521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575921.045526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575921.045530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575921.045533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575921.045536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575921.045539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575921.045542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575921.941497:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575921.941504:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575921.941509:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575921.941510:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575921.941513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575921.941514:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575921.941580:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575921.941584:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575921.941587:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575922.069519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575922.069525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575922.069527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575922.069533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575922.069537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575922.069547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575922.069550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575922.069553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575922.069555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575922.965505:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575922.965511:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575922.965516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575922.965518:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575922.965521:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575922.965523:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575922.965595:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575922.965599:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575922.965603:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575923.093487:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575923.093494:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575923.093496:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575923.093501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575923.093504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575923.093508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575923.093511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575923.093513:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575923.093516:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575923.605515:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575923.605525:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575923.605528:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575923.605532:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575923.605536:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575923.605543:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12854
00000400:00000200:3.0:1788575923.605551:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575923.605557:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575923.605560:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575923.605563:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575923.605566:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575923.605568:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575923.605571:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575923.605574:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575923.605576:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575923.605579:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575923.605582:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575923.605584:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575923.605586:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575923.605599:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575923.605602:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575923.605604:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575923.605606:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575923.605610:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575923.605613:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575923.605616:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575923.605618:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575923.605622:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575923.605625:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575923.605627:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575923.605629:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575923.605639:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575923.605642:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575923.605646:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575923.605647:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575923.605650:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575923.605653:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575923.605657:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575923.605664:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575923.605670:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575923.605679:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575923.605688:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575923.605693:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575923.605697:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575923.605723:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575923.605726:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575923.605729:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575923.605730:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575923.605732:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575923.605734:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575923.605737:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12854
00000400:00000200:3.0:1788575923.605740:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575923.605743:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575923.605745:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575923.605748:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575923.605750:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575923.605752:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575923.605755:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575923.605757:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575923.605759:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575923.605761:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575923.605763:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575923.605765:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575923.605767:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788575923.605768:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1788575923.605770:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575923.605774:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788575923.605775:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575923.605777:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575923.605779:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575923.605780:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575923.605784:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575923.605789:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:0.0:1788575923.605790:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575923.605791:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575923.605795:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575923.605800:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575923.605804:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1788575923.605804:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575923.605808:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575923.605808:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1788575923.605814:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:3.0:1788575923.605815:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788575923.605819:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1788575923.605820:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575923.605824:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575923.605825:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575923.605829:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000100:00100000:0.0:1788575923.605831:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393245824:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1788575923.605833:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575923.605838:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1788575923.605840:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788575923.605841:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575923.605844:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575923.605846:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575923.605846:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575923.605848:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575923.605850:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575923.605853:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788575923.605855:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575923.605859:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575923.605863:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575923.605867:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575923.605868:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788575923.605877:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575923.605884:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000100:00000001:3.0:1788575923.605889:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1788575923.605893:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000001:3.0:1788575923.605893:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575923.605901:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575923.605903:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393245824, portal 25
00000100:00000001:0.0:1788575923.605908:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575923.605911:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000001:3.0:1788575923.605914:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575923.605918:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575923.605922:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575923.605923:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575923.605926:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575923.605927:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575923.605930:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575923.605933:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12854
00000400:00000200:3.0:1788575923.605937:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575923.605942:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575923.605946:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575923.605952:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575923.605953:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575923.605956:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575923.605960:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575923.605961:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:0.0:1788575923.605966:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000001:3.0:1788575923.605966:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575923.605971:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000200:0.0:1788575923.605973:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393245824, offset 0
00000020:00000040:3.0:1788575923.605973:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575923.605977:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575923.605980:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575923.605982:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788575923.605985:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:3.0:1788575923.605986:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575923.605992:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575923.605994:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788575923.605997:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788575923.605998:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575923.606003:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575923.606006:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:0.0:1788575923.606008:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:3.0:1788575923.606012:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575923.606016:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575923.606017:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1788575923.606019:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1788575923.606024:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1788575923.606024:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575923.606028:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575923.606032:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575923.606033:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1788575923.606037:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000008:00100000:3.0:1788575923.606038:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:0.0:1788575923.606042:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331176:331176:256:4294967295] 192.168.202.110@tcp LPNI seq info [331133:331133:8:4294967295]
00000008:00000001:3.0:1788575923.606049:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575923.606051:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575923.606054:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575923.606059:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000020:00000040:3.0:1788575923.606064:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:0.0:1788575923.606066:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1788575923.606068:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575923.606071:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575923.606074:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575923.606077:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:0.0:1788575923.606078:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575923.606080:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788575923.606086:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
02000000:00000010:3.0:1788575923.606087:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788575923.606092:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788575923.606097:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:3.0:1788575923.606101:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1788575923.606103:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788575923.606115:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788575923.606116:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575923.606120:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788575923.606121:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:3.0:1788575923.606127:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:0.0:1788575923.606132:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575923.606136:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000100:00000001:3.0:1788575923.606142:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575923.606145:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1788575923.606146:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575923.606151:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:0.0:1788575923.606153:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1788575923.606158:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12864)
00000400:00000200:0.0:1788575923.606160:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575923.606164:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575923.606169:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575923.606174:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606183:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575923.606186:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606193:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575923.606194:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575923.606199:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575923.606203:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575923.606206:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307528121
00000100:00000010:0.0:1788575923.606211:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788575923.606245:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788575923.606250:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307528121
00000100:00000010:0.0:1788575923.606253:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ba0.
00000100:00000001:0.0:1788575923.606256:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788575923.606259:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575923.606262:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575923.606264:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575923.606269:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575923/real 1788575923]  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575923.606284:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575923.606285:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575923.606287:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575923.606290:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575923.606292:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575923.606294:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575923.606298:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575923.606305:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606311:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575923.606313:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606315:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575923.606318:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575923.606322:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606328:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575923.606333:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606339:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575923.606342:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606344:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575923.606345:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606347:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575923.606349:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575923.606352:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575923.606355:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575923.606358:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575923.606362:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575923.606371:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393245824:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575923.606375:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606377:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575923.606380:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393245824/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606386:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575923.606389:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575923.606422:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575923.606426:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575923.606428:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575923.606429:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788575923.606432:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575923.606435:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788575923.606439:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575923.606441:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575923.606442:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:0.0:1788575923.606446:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575923.606447:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606449:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606456:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575923.606458:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575923.606460:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575923.606461:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575923.606462:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575923.606466:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606473:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575923.606474:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575923.606476:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575923.606478:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575923.606481:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575923.606484:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245952:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575923.606487:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575923.606489:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575923.606491:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575923.606492:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575923.606495:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575923.606497:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575923.606506:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575923.606509:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575923.606513:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575923.606516:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393245952, portal 10
00000100:00000001:0.0:1788575923.606518:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575923.606519:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:0.0:1788575923.606526:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606532:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575923.606534:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575923.606537:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575923.606539:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393245952, offset 0
00000400:00000200:0.0:1788575923.606543:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575923.606547:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575923.606551:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575923.606553:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575923.606556:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575923.606558:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575923.606561:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575923.606564:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331177:331177:256:4294967295] 192.168.202.110@tcp LPNI seq info [331134:331134:8:4294967295]
00000400:00000200:0.0:1788575923.606568:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575923.606574:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575923.606578:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575923.606582:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575923.606585:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575923.606588:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788575923.606592:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575923.606594:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788575923.606598:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575923.606600:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575923.606603:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575923.606606:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575923.606609:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575923.606611:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575923.606613:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575923.606617:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606624:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575923.606627:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606632:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575923.606634:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575923.606637:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575923.606641:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788575923.606644:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575923.606646:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307527207
00000100:00000001:0.0:1788575923.606649:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307528121
00000100:00000001:0.0:1788575923.606651:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307527207
00000100:00000001:0.0:1788575923.606652:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307528121
00000100:00000001:0.0:1788575923.606654:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575923.606656:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575923.606658:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575923.606662:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575923/real 1788575923]  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575923.606669:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575923.606670:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575923.606672:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575923.606675:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575923.606676:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575923.606678:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575923.606681:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575923.606688:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606694:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575923.606695:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606697:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575923.606699:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575923.606702:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606708:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575923.606712:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606719:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575923.606721:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606723:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575923.606724:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606726:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575923.606728:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575923.606731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575923.606734:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575923.606736:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575923.606740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575923.606748:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393245952:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575923.606752:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575923.606755:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606761:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575923.606762:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575923.606764:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575923.606766:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575923.606768:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575923.606770:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246080:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575923.606773:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575923.606775:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575923.606777:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575923.606778:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575923.606781:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575923.606783:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575923.606787:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575923.606790:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575923.606793:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575923.606796:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393246080, portal 4
00000100:00000001:0.0:1788575923.606797:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575923.606798:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:0.0:1788575923.606804:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606810:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575923.606812:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575923.606815:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575923.606817:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393246080, offset 0
00000400:00000200:0.0:1788575923.606820:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575923.606824:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575923.606827:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575923.606830:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575923.606833:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575923.606835:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575923.606837:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575923.606840:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331178:331178:256:4294967295] 192.168.202.110@tcp LPNI seq info [331135:331135:8:4294967295]
00000400:00000200:0.0:1788575923.606845:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575923.606850:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575923.606855:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575923.606858:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575923.606862:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575923.606865:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788575923.606868:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575923.606871:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788575923.606875:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575923.606876:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575923.606879:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575923.606884:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575923.606886:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575923.606888:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575923.606891:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575923.606894:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606901:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575923.606904:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606911:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575923.606912:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575923.606915:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575923.606918:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788575923.606922:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575923.606924:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307527353
00000100:00000001:0.0:1788575923.606926:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307528121
00000100:00000001:0.0:1788575923.606928:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307527353
00000100:00000001:0.0:1788575923.606930:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307528121
00000100:00000001:0.0:1788575923.606932:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575923.606934:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575923.606936:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575923.606940:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575923/real 1788575923]  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575923.606948:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575923.606949:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575923.606951:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575923.606953:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575923.606955:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575923.606957:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575923.606960:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575923.606968:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606975:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575923.606977:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.606978:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575923.606980:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575923.606983:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.606990:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575923.606994:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.607001:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575923.607003:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.607005:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575923.607006:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.607008:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575923.607010:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575923.607013:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575923.607016:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575923.607018:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575923.607022:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575923.607030:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246080:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575923.607033:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.607035:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575923.607038:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393245952/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.607044:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575923.607046:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575923.607047:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575923.607050:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575923.607051:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575923.607053:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788575923.607055:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575923.607056:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788575923.607059:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575923.607060:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575923.607061:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:0.0:1788575923.607064:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575923.607065:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.607067:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575923.607070:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393246080/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575923.607076:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575923.607077:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575923.607078:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575923.607081:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575923.607082:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575923.607083:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788575923.607085:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575923.607086:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788575923.607088:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575923.607089:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575923.607090:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:0.0:1788575923.607093:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575923.607094:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.607095:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575923.607099:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575923.607100:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575923.607102:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575923.607104:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575923.989492:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575923.989499:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575923.989505:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575923.989506:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575923.989510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575923.989512:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575923.989587:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575923.989590:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575923.989594:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575924.118332:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575924.118337:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575924.118340:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575924.118344:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575924.118349:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575924.118352:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575924.118355:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575924.118358:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575924.118361:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575924.565507:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307528121, at 4307528128
00000100:00000010:1.0:1788575924.565521:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ba0.
00000100:00000040:0.0:1788575924.565629:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307528121, at 4307528128
00000100:00000010:0.0:1788575924.565635:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00010000:00000001:3.0:1788575925.013470:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575925.013476:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575925.013482:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575925.013483:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575925.013487:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575925.013489:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575925.013608:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575925.013611:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575925.013615:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575925.143127:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575925.143132:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575925.143134:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575925.143146:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575925.143152:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575925.143155:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575925.143158:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575925.143160:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575925.143164:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575926.037513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575926.037529:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575926.037535:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575926.037538:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575926.037540:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575926.037543:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575926.037546:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575926.037549:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575926.037551:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575926.037553:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575926.037556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575926.037557:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575926.037559:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575926.037560:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575926.037561:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575926.037562:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575926.037564:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575926.037566:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575926.037635:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575926.037640:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575926.037643:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575926.165587:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575926.165593:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575926.165595:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575926.165599:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575926.165601:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575926.165604:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575926.165606:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575926.165607:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575926.165609:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575927.061522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575927.061527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575927.061530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575927.061531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575927.061534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575927.061535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575927.061598:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575927.061602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575927.061605:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575927.189510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575927.189517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575927.189519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575927.189525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575927.189529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575927.189533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575927.189536:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575927.189538:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575927.189541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575928.085512:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575928.085518:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575928.085522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575928.085523:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575928.085527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575928.085528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575928.085593:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575928.085599:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575928.085604:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575928.213470:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575928.213484:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575928.213486:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575928.213491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575928.213495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575928.213499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575928.213502:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575928.213504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575928.213506:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575928.725509:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575928.725519:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575928.725522:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575928.725527:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575928.725532:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575928.725540:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12859
00000400:00000200:3.0:1788575928.725550:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575928.725556:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575928.725560:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575928.725564:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575928.725567:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575928.725569:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575928.725572:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575928.725577:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575928.725579:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575928.725582:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575928.725586:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575928.725589:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575928.725591:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575928.725595:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575928.725598:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575928.725600:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575928.725603:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575928.725606:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575928.725610:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575928.725613:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575928.725616:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575928.725619:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575928.725622:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575928.725625:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575928.725628:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575928.725638:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000020:00000040:3.0:1788575928.725641:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575928.725645:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575928.725647:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575928.725650:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575928.725652:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575928.725655:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575928.725663:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788575928.725669:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575928.725678:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575928.725687:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575928.725691:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575928.725696:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575928.725719:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575928.725723:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575928.725730:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575928.725731:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575928.725734:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575928.725737:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575928.725740:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12859
00000400:00000200:3.0:1788575928.725744:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575928.725747:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575928.725750:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575928.725753:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575928.725755:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575928.725758:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575928.725760:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788575928.725760:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575928.725765:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575928.725767:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:0.0:1788575928.725768:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575928.725769:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788575928.725772:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788575928.725773:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575928.725777:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575928.725781:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575928.725783:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575928.725787:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575928.725792:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575928.725797:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575928.725802:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788575928.725803:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:3.0:1788575928.725805:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788575928.725808:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575928.725809:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1788575928.725814:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788575928.725815:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575928.725818:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575928.725819:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575928.725823:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575928.725825:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1788575928.725828:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393246208:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788575928.725829:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575928.725833:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575928.725836:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:3.0:1788575928.725838:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788575928.725842:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:3.0:1788575928.725843:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575928.725846:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575928.725847:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575928.725852:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:3.0:1788575928.725854:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1180.
02000000:00000010:0.0:1788575928.725859:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
00000020:00000040:3.0:1788575928.725859:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575928.725864:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575928.725864:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575928.725867:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575928.725870:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1788575928.725873:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788575928.725873:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575928.725875:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788575928.725879:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
02000000:00000010:3.0:1788575928.725882:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000400:00000010:0.0:1788575928.725886:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000001:3.0:1788575928.725888:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788575928.725892:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393246208, portal 25
00000100:00000001:0.0:1788575928.725896:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00100000:3.0:1788575928.725897:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575928.725898:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:0.0:1788575928.725906:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575928.725913:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575928.725917:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575928.725920:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575928.725921:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575928.725923:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393246208, offset 0
00000100:00000001:3.0:1788575928.725927:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575928.725928:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788575928.725936:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca1dca9a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788575928.725937:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575928.725945:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575928.725953:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788575928.725953:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575928.725959:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:3.0:1788575928.725959:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788575928.725964:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1788575928.725966:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1788575928.725968:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575928.725970:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1788575928.725974:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331179:331179:256:4294967295] 192.168.202.110@tcp LPNI seq info [331136:331136:8:4294967295]
00000100:00080000:3.0:1788575928.725975:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1788575928.725980:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575928.725981:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575928.725987:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12859
00000400:00000200:0.0:1788575928.725990:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1788575928.725995:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:0.0:1788575928.725997:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788575928.726001:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575928.726005:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000010:0.0:1788575928.726008:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000100:00000040:3.0:1788575928.726008:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575928.726013:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1788575928.726016:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575928.726018:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000200:0.0:1788575928.726020:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:3.0:1788575928.726023:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575928.726028:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.1:1788575928.726028:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1788575928.726031:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000800:00000200:0.1:1788575928.726032:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:3.0:1788575928.726035:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000100:0.0:1788575928.726038:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1788575928.726039:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000010:0.0:1788575928.726042:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000020:00000001:3.0:1788575928.726043:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575928.726047:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788575928.726049:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1788575928.726053:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788575928.726055:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1788575928.726058:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788575928.726061:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000100:00000040:3.0:1788575928.726063:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:0.0:1788575928.726064:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:3.0:1788575928.726068:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788575928.726070:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000020:00000040:3.0:1788575928.726072:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000200:0.0:1788575928.726075:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575928.726078:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575928.726085:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726088:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1788575928.726089:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1788575928.726093:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1788575928.726094:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575928.726099:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575928.726104:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575928.726106:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726108:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575928.726113:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575928.726113:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575928.726119:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575928.726121:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575928.726122:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726126:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307533241
00000100:00000010:3.0:1788575928.726128:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000100:00000010:0.0:1788575928.726133:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000020:00000040:3.0:1788575928.726133:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575928.726135:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575928.726138:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575928.726139:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575928.726140:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575928.726142:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726143:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000010:3.0:1788575928.726146:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000100:00000001:0.0:1788575928.726149:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307533241
00000100:00000001:3.0:1788575928.726150:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575928.726155:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7b40.
00000100:00100000:3.0:1788575928.726157:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575928.726159:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788575928.726164:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575928.726182:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726184:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575928.726189:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575928/real 1788575928]  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575928.726190:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000001:0.0:1788575928.726199:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1788575928.726199:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1788575928.726203:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575928.726206:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000100:00000040:3.0:1788575928.726208:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788575928.726211:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575928.726215:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575928.726220:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000001:3.0:1788575928.726224:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788575928.726228:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575928.726229:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575928.726237:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575928.726245:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12869)
00000100:00000200:0.0:1788575928.726246:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726255:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575928.726257:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726260:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575928.726262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575928.726266:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726272:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575928.726276:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726282:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575928.726285:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726286:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575928.726287:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726289:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575928.726291:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575928.726294:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575928.726297:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575928.726299:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575928.726303:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575928.726310:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393246208:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575928.726315:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726317:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575928.726319:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393246208/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726331:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575928.726333:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575928.726336:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575928.726339:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575928.726341:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575928.726342:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788575928.726344:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575928.726346:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:0.0:1788575928.726350:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575928.726351:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575928.726353:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788575928.726357:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575928.726358:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726360:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726367:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575928.726368:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726370:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575928.726371:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575928.726373:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575928.726376:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726382:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575928.726383:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575928.726385:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575928.726387:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575928.726389:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575928.726418:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246336:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575928.726422:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575928.726424:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575928.726425:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575928.726427:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575928.726430:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575928.726432:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575928.726435:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575928.726439:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575928.726442:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575928.726445:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393246336, portal 10
00000100:00000001:0.0:1788575928.726447:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575928.726448:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000100:00000040:0.0:1788575928.726455:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726461:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575928.726464:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575928.726466:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575928.726468:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393246336, offset 0
00000400:00000200:0.0:1788575928.726471:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575928.726474:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575928.726478:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575928.726480:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575928.726483:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575928.726485:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575928.726487:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575928.726490:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331180:331180:256:4294967295] 192.168.202.110@tcp LPNI seq info [331137:331137:8:4294967295]
00000400:00000200:0.0:1788575928.726494:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575928.726499:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575928.726504:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575928.726507:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575928.726510:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575928.726513:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788575928.726517:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575928.726520:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788575928.726523:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575928.726525:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575928.726528:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575928.726531:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575928.726534:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575928.726536:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575928.726538:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575928.726542:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726549:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575928.726552:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726558:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726560:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575928.726563:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575928.726566:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788575928.726569:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575928.726571:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307532327
00000100:00000001:0.0:1788575928.726574:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307533241
00000100:00000001:0.0:1788575928.726576:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307532327
00000100:00000001:0.0:1788575928.726578:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307533241
00000100:00000001:0.0:1788575928.726580:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575928.726582:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726584:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575928.726587:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575928/real 1788575928]  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575928.726594:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575928.726595:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575928.726597:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575928.726600:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575928.726601:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575928.726603:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575928.726606:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575928.726614:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726625:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575928.726627:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575928.726629:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575928.726633:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726640:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575928.726644:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726650:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575928.726653:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726654:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575928.726655:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726657:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575928.726659:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575928.726661:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575928.726664:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575928.726666:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575928.726670:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575928.726677:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246336:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575928.726681:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575928.726684:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726689:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575928.726691:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575928.726693:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575928.726695:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575928.726696:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575928.726699:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575928.726701:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575928.726703:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575928.726704:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575928.726706:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575928.726709:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575928.726710:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575928.726714:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575928.726716:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575928.726719:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575928.726721:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393246464, portal 4
00000100:00000001:0.0:1788575928.726723:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575928.726725:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:0.0:1788575928.726730:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726736:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575928.726738:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575928.726740:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575928.726742:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393246464, offset 0
00000400:00000200:0.0:1788575928.726745:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575928.726748:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575928.726751:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575928.726754:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575928.726756:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575928.726758:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575928.726760:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575928.726763:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331181:331181:256:4294967295] 192.168.202.110@tcp LPNI seq info [331138:331138:8:4294967295]
00000400:00000200:0.0:1788575928.726767:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575928.726773:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575928.726778:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575928.726780:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575928.726784:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575928.726787:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1788575928.726791:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575928.726793:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1788575928.726798:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575928.726799:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575928.726802:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575928.726805:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575928.726808:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575928.726809:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575928.726811:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575928.726815:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726822:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575928.726825:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726831:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726833:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575928.726835:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575928.726838:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788575928.726841:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575928.726843:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307532473
00000100:00000001:0.0:1788575928.726845:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307533241
00000100:00000001:0.0:1788575928.726847:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307532473
00000100:00000001:0.0:1788575928.726849:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307533241
00000100:00000001:0.0:1788575928.726851:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575928.726853:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.726854:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575928.726857:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575928/real 1788575928]  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575928.726864:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575928.726865:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575928.726867:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575928.726869:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575928.726870:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575928.726872:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575928.726875:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575928.726882:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726888:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575928.726890:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726891:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575928.726893:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575928.726896:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726902:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575928.726906:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726911:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575928.726913:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726915:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575928.726916:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726918:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575928.726920:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575928.726922:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575928.726924:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575928.726927:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575928.726931:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575928.726938:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575928.726942:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726944:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575928.726946:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393246336/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726952:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575928.726954:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575928.726956:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575928.726958:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575928.726960:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575928.726961:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788575928.726963:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575928.726964:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788575928.726967:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575928.726968:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575928.726969:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:0.0:1788575928.726971:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575928.726972:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.726974:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575928.726977:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393246464/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575928.726983:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575928.726984:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575928.726985:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575928.726987:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575928.726988:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575928.726989:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788575928.726990:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575928.726991:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:0.0:1788575928.726993:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575928.726994:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575928.726995:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:0.0:1788575928.726997:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575928.726999:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.727000:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575928.727003:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575928.727004:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575928.727006:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575928.727008:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575929.109505:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575929.109514:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575929.109516:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575929.109519:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575929.109523:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575929.109526:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575929.109528:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575929.109530:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575929.109533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575929.109534:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575929.109538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575929.109539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575929.109600:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575929.109604:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575929.109607:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575929.237476:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575929.237483:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575929.237484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575929.237489:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575929.237492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575929.237495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575929.237497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575929.237498:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575929.237501:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575929.685498:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307533241, at 4307533248
00000100:00000040:1.0:1788575929.685498:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307533241, at 4307533248
00000100:00000010:0.0:1788575929.685514:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:1.0:1788575929.685516:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7b40.
00010000:00000001:3.0:1788575930.133470:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575930.133477:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575930.133482:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575930.133484:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575930.133488:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575930.133490:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575930.133549:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575930.133555:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575930.133558:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575930.261509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575930.261517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575930.261518:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575930.261522:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575930.261525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575930.261528:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575930.261530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575930.261531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575930.261533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575931.157604:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575931.157610:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575931.157616:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575931.157617:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575931.157621:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575931.157622:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575931.157686:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575931.157690:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575931.157693:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575931.285641:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575931.285645:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575931.285646:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575931.285650:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575931.285654:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575931.285656:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575931.285658:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575931.285660:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575931.285662:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575932.181486:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575932.181504:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575932.181510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575932.181511:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575932.181515:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575932.181516:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575932.181568:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575932.181571:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575932.181573:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575932.309503:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575932.309510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575932.309512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575932.309517:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575932.309520:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575932.309524:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575932.309527:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575932.309530:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575932.309532:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575933.205505:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575933.205511:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575933.205516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575933.205517:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575933.205521:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575933.205522:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575933.205574:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575933.205578:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575933.205582:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575933.333598:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575933.333606:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575933.333608:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575933.333613:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575933.333617:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575933.333621:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575933.333623:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575933.333626:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575933.333629:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575933.845483:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575933.845494:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575933.845497:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575933.845501:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575933.845506:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575933.845514:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12864
00000400:00000200:3.0:1788575933.845522:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575933.845528:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575933.845530:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575933.845534:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845537:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575933.845539:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845541:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575933.845545:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575933.845547:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575933.845550:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575933.845553:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575933.845555:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575933.845558:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575933.845561:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845562:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575933.845564:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845566:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575933.845569:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575933.845573:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845575:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575933.845578:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575933.845581:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575933.845584:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575933.845586:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575933.845589:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575933.845597:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788575933.845601:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575933.845605:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575933.845606:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575933.845609:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575933.845612:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575933.845616:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575933.845625:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:3.0:1788575933.845632:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575933.845641:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575933.845651:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575933.845656:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575933.845661:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575933.845683:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575933.845686:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575933.845689:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575933.845691:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575933.845692:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575933.845695:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575933.845697:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12864
00000400:00000200:3.0:1788575933.845701:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575933.845704:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575933.845706:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575933.845709:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845712:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575933.845714:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845716:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575933.845719:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575933.845721:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575933.845723:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575933.845725:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575933.845727:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788575933.845729:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575933.845729:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788575933.845735:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1788575933.845735:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575933.845739:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1788575933.845740:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575933.845743:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845748:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:0.0:1788575933.845749:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575933.845751:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575933.845757:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845763:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575933.845767:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788575933.845768:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575933.845772:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575933.845773:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575933.845776:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575933.845777:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575933.845780:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575933.845781:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1788575933.845786:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1788575933.845787:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00100000:0.0:1788575933.845792:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393246592:192.168.202.110@tcp:250:kworker.0
00000008:00000001:3.0:1788575933.845792:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575933.845796:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575933.845800:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:3.0:1788575933.845802:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788575933.845805:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575933.845808:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575933.845809:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575933.845812:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575933.845814:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788575933.845814:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575933.845817:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575933.845818:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575933.845824:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788575933.845825:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8c00.
00000100:00000001:3.0:1788575933.845830:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575933.845836:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
00000100:00100000:3.0:1788575933.845837:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1788575933.845842:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575933.845850:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1788575933.845852:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575933.845854:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788575933.845858:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000100:00000040:3.0:1788575933.845862:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575933.845872:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788575933.845875:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00080000:3.0:1788575933.845876:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575933.845880:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575933.845883:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575933.845885:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:0.0:1788575933.845889:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393246592, portal 25
00000100:00000001:3.0:1788575933.845889:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575933.845893:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12864
00000100:00000001:0.0:1788575933.845898:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575933.845900:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:0.0:1788575933.845908:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788575933.845909:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575933.845918:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:3.0:1788575933.845919:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575933.845924:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575933.845925:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575933.845930:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000040:3.0:1788575933.845931:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:0.0:1788575933.845934:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393246592, offset 0
00000100:00000001:3.0:1788575933.845935:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575933.845940:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845944:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788575933.845945:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:3.0:1788575933.845949:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575933.845951:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575933.845953:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1788575933.845956:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1788575933.845957:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575933.845961:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575933.845963:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788575933.845966:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1788575933.845969:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845972:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788575933.845975:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1788575933.845976:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788575933.845980:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575933.845980:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575933.845983:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:0.0:1788575933.845987:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1788575933.845988:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1788575933.845993:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788575933.845993:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575933.845996:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1788575933.846001:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331182:331182:256:4294967295] 192.168.202.110@tcp LPNI seq info [331139:331139:8:4294967295]
02000000:00000001:3.0:1788575933.846002:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575933.846007:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575933.846011:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575933.846012:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575933.846015:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575933.846018:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575933.846022:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575933.846027:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000400:00000200:0.0:1788575933.846029:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:3.0:1788575933.846033:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:0.0:1788575933.846038:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575933.846038:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575933.846041:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575933.846044:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:0.0:1788575933.846046:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
02000000:00000001:3.0:1788575933.846046:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575933.846049:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788575933.846055:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:3.0:1788575933.846057:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000800:00000200:0.0:1788575933.846060:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:3.0:1788575933.846062:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788575933.846069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:3.0:1788575933.846071:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1788575933.846073:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788575933.846083:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575933.846087:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000100:00000001:3.0:1788575933.846087:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575933.846092:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575933.846096:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1788575933.846098:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788575933.846104:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575933.846113:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000100:00000001:3.0:1788575933.846114:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788575933.846117:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575933.846133:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:3.0:1788575933.846135:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:0.0:1788575933.846139:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788575933.846141:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575933.846149:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12875)
00000100:00000001:0.0:1788575933.846164:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575933.846168:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846176:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575933.846183:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575933.846189:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575933.846193:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575933.846196:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307538361
00000100:00000010:0.0:1788575933.846201:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788575933.846206:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788575933.846218:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307538361
00000100:00000010:0.0:1788575933.846221:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7d20.
00000100:00000001:0.0:1788575933.846224:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788575933.846227:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575933.846230:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575933.846232:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575933.846237:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575933/real 1788575933]  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575933.846246:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575933.846248:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575933.846249:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575933.846259:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575933.846261:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575933.846264:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575933.846267:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575933.846276:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846281:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575933.846283:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846285:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575933.846288:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575933.846291:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846298:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575933.846302:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846308:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575933.846311:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846312:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575933.846313:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846315:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575933.846322:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575933.846325:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575933.846328:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575933.846331:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575933.846334:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575933.846342:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393246592:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575933.846347:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846348:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575933.846351:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393246592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846357:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575933.846359:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575933.846362:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575933.846366:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575933.846367:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575933.846369:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788575933.846372:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575933.846374:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:0.0:1788575933.846377:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575933.846379:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575933.846380:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:0.0:1788575933.846384:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575933.846385:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846387:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846424:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575933.846427:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575933.846429:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575933.846437:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575933.846438:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575933.846442:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846449:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575933.846451:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575933.846453:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575933.846455:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575933.846457:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575933.846460:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575933.846463:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575933.846465:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575933.846466:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575933.846467:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575933.846470:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575933.846471:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575933.846473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575933.846475:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575933.846478:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575933.846480:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393246720, portal 10
00000100:00000001:0.0:1788575933.846481:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575933.846482:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:0.0:1788575933.846486:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846491:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575933.846492:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575933.846493:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575933.846495:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393246720, offset 0
00000400:00000200:0.0:1788575933.846497:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575933.846499:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575933.846502:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575933.846504:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575933.846505:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575933.846507:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575933.846508:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575933.846510:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331183:331183:256:4294967295] 192.168.202.110@tcp LPNI seq info [331140:331140:8:4294967295]
00000400:00000200:0.0:1788575933.846513:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575933.846517:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575933.846520:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575933.846522:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575933.846524:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575933.846526:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788575933.846529:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575933.846531:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788575933.846533:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575933.846534:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575933.846536:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575933.846539:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575933.846541:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575933.846542:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575933.846543:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575933.846545:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846550:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575933.846552:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846559:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575933.846560:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575933.846562:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575933.846564:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788575933.846566:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575933.846567:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307537447
00000100:00000001:0.0:1788575933.846569:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307538361
00000100:00000001:0.0:1788575933.846571:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307537447
00000100:00000001:0.0:1788575933.846572:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307538361
00000100:00000001:0.0:1788575933.846573:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575933.846575:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575933.846576:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575933.846578:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575933/real 1788575933]  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575933.846582:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575933.846583:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575933.846584:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575933.846585:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575933.846586:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575933.846587:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575933.846589:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575933.846593:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846596:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575933.846597:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846598:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575933.846599:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575933.846601:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846614:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575933.846618:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846624:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575933.846627:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846629:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575933.846630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846632:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575933.846634:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575933.846636:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575933.846639:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575933.846642:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575933.846646:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575933.846654:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575933.846657:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575933.846659:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846663:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575933.846664:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575933.846665:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575933.846666:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575933.846667:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575933.846669:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246848:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575933.846671:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575933.846672:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575933.846673:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575933.846674:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575933.846676:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788575933.846677:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575933.846680:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575933.846681:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575933.846684:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575933.846686:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393246848, portal 4
00000100:00000001:0.0:1788575933.846687:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575933.846688:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:0.0:1788575933.846691:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846695:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575933.846696:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575933.846697:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575933.846699:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393246848, offset 0
00000400:00000200:0.0:1788575933.846702:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575933.846706:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575933.846709:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575933.846712:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575933.846714:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575933.846716:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575933.846718:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575933.846721:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331184:331184:256:4294967295] 192.168.202.110@tcp LPNI seq info [331141:331141:8:4294967295]
00000400:00000200:0.0:1788575933.846725:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575933.846731:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575933.846735:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575933.846738:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575933.846742:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575933.846745:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788575933.846748:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575933.846749:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788575933.846751:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575933.846753:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575933.846755:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575933.846757:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575933.846759:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575933.846760:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575933.846762:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575933.846764:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846769:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575933.846770:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846774:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575933.846775:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575933.846777:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575933.846779:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788575933.846781:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575933.846782:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307537593
00000100:00000001:0.0:1788575933.846784:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307538361
00000100:00000001:0.0:1788575933.846785:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307537593
00000100:00000001:0.0:1788575933.846786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307538361
00000100:00000001:0.0:1788575933.846787:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575933.846789:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575933.846790:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575933.846791:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575933/real 1788575933]  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575933.846796:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575933.846797:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575933.846798:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575933.846800:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575933.846801:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575933.846804:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575933.846807:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575933.846814:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846819:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575933.846821:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846823:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575933.846824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575933.846827:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846834:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575933.846838:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846845:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575933.846847:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846848:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575933.846849:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846851:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575933.846853:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575933.846855:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575933.846859:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575933.846861:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575933.846864:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575933.846872:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393246848:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575933.846875:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846878:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575933.846881:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393246720/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846888:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575933.846890:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575933.846892:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788575933.846893:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575933.846894:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575933.846895:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788575933.846896:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575933.846898:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:0.0:1788575933.846899:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575933.846900:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575933.846901:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:0.0:1788575933.846902:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575933.846903:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846904:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575933.846907:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393246848/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575933.846910:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575933.846911:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575933.846912:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788575933.846913:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575933.846914:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575933.846914:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788575933.846915:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575933.846916:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:0.0:1788575933.846917:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575933.846918:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575933.846918:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:0.0:1788575933.846920:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575933.846921:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846921:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575933.846924:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575933.846925:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575933.846926:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575933.846927:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575934.229518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575934.229524:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575934.229529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575934.229530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575934.229533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575934.229534:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575934.229596:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575934.229599:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575934.229603:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575934.357835:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575934.357843:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575934.357845:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575934.357850:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575934.357853:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575934.357856:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575934.357859:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575934.357860:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575934.357862:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575934.805522:0:216430:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307538361, at 4307538368
00000100:00000040:1.0:1788575934.805522:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307538361, at 4307538368
00000100:00000010:1.0:1788575934.805538:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d20.
00000100:00000010:0.0:1788575934.805545:0:216430:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00010000:00000001:3.0:1788575935.253662:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575935.253667:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575935.253671:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575935.253672:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575935.253674:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575935.253675:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575935.253728:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575935.253732:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575935.253735:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575935.381510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575935.381517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575935.381520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575935.381524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575935.381528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575935.381532:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575935.381535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575935.381537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575935.381540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575936.277519:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575936.277525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575936.277530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575936.277534:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575936.277536:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575936.277539:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575936.277542:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575936.277544:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575936.277546:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575936.277548:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575936.277550:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575936.277551:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575936.277552:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575936.277553:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575936.277554:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575936.277555:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575936.277556:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575936.277557:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575936.277619:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575936.277623:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575936.277626:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575936.405567:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575936.405576:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575936.405579:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575936.405585:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575936.405590:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575936.405594:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575936.405597:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575936.405599:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575936.405602:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575937.301519:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575937.301526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575937.301532:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575937.301533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575937.301536:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575937.301537:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575937.301603:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575937.301606:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575937.301610:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575937.429551:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575937.429561:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575937.429564:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575937.429571:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575937.429575:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575937.429581:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575937.429584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575937.429587:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575937.429592:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575938.328248:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575938.328257:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575938.328264:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575938.328266:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575938.328270:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575938.328271:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575938.328338:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575938.328342:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575938.328346:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575938.453512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575938.453523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575938.453526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575938.453535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575938.453539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575938.453545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.453549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575938.453552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.453556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575938.965565:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575938.965580:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575938.965583:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575938.965587:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575938.965592:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575938.965599:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12870
00000400:00000200:0.0:1788575938.965608:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575938.965617:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575938.965620:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575938.965623:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575938.965626:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575938.965629:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575938.965632:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575938.965637:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575938.965639:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575938.965642:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575938.965646:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575938.965649:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575938.965651:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575938.965654:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575938.965656:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575938.965657:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575938.965658:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575938.965662:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575938.965664:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575938.965666:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575938.965669:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575938.965671:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575938.965674:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575938.965675:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575938.965684:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575938.965696:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788575938.965700:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575938.965705:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575938.965706:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575938.965710:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575938.965713:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575938.965719:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575938.965728:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788575938.965742:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575938.965752:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575938.965762:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575938.965771:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575938.965776:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575938.965807:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575938.965811:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575938.965814:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575938.965816:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575938.965818:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575938.965821:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575938.965824:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12870
00000400:00000200:0.0:1788575938.965829:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575938.965833:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575938.965836:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575938.965839:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575938.965842:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575938.965842:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1788575938.965846:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575938.965849:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575938.965853:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788575938.965855:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575938.965860:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575938.965862:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000040:2.0:1788575938.965863:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1788575938.965865:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575938.965868:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575938.965873:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575938.965877:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575938.965881:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1788575938.965885:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575938.965886:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575938.965889:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575938.965892:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1788575938.965894:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575938.965897:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575938.965898:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575938.965902:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575938.965904:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00100000:2.0:1788575938.965910:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393246976:192.168.202.110@tcp:250:kworker.0
00000100:00080000:0.0:1788575938.965912:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575938.965919:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575938.965919:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575938.965923:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575938.965931:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575938.965931:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575938.965936:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575938.965936:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575938.965942:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575938.965942:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1788575938.965949:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1788575938.965950:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
00000008:00000001:0.0:1788575938.965953:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575938.965955:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575938.965956:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575938.965962:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788575938.965964:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:2.0:1788575938.965967:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575938.965968:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575938.965971:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575938.965974:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1788575938.965976:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
02000000:00000001:0.0:1788575938.965977:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575938.965980:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575938.965987:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000400:00000010:2.0:1788575938.965987:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000001:0.0:1788575938.965994:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575938.965998:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393246976, portal 25
00000100:00100000:0.0:1788575938.966002:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575938.966003:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575938.966007:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000001:0.0:1788575938.966016:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1788575938.966018:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575938.966020:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575938.966029:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966036:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1788575938.966041:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575938.966044:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788575938.966045:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575938.966048:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:2.0:1788575938.966049:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000001:0.0:1788575938.966050:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575938.966053:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:2.0:1788575938.966056:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393246976, offset 0
00000100:00000001:0.0:1788575938.966057:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575938.966063:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12870
00000400:00000200:2.0:1788575938.966068:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575938.966069:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575938.966097:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575938.966105:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575938.966109:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:0.0:1788575938.966110:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575938.966117:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575938.966119:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575938.966120:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575938.966124:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575938.966127:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1788575938.966129:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575938.966132:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575938.966134:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:2.0:1788575938.966135:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:0.0:1788575938.966136:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575938.966140:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575938.966143:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575938.966143:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:0.0:1788575938.966147:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788575938.966152:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331185:331185:256:4294967295] 192.168.202.110@tcp LPNI seq info [331142:331142:8:4294967295]
00000100:00000040:0.0:1788575938.966153:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575938.966157:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575938.966160:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575938.966163:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575938.966164:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1788575938.966166:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575938.966169:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575938.966179:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788575938.966180:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575938.966187:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:2.0:1788575938.966187:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575938.966193:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:2.0:1788575938.966193:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
02000000:00000001:0.0:1788575938.966196:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575938.966199:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575938.966202:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:2.0:1788575938.966204:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000008:00000001:0.0:1788575938.966206:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575938.966210:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575938.966210:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000010:0.0:1788575938.966216:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000800:00000200:2.1:1788575938.966220:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1788575938.966222:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.1:1788575938.966227:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
02000000:00000001:0.0:1788575938.966228:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575938.966231:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575938.966235:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000100:2.0:1788575938.966237:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575938.966238:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575938.966243:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575938.966243:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
02000000:00000010:0.0:1788575938.966250:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575938.966255:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575938.966259:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:0.0:1788575938.966261:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575938.966268:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1788575938.966276:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575938.966278:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000100:00000001:0.0:1788575938.966279:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575938.966283:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000040:0.0:1788575938.966287:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966291:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:0.0:1788575938.966300:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575938.966300:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575938.966305:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575938.966314:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575938.966319:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575938.966322:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12880)
00000100:00000040:2.0:1788575938.966326:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966335:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575938.966337:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575938.966345:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575938.966350:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575938.966353:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307543481
00000100:00000010:2.0:1788575938.966359:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cba0.
00000100:00000001:2.0:1788575938.966368:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575938.966378:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307543481
00000100:00000010:2.0:1788575938.966381:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cc60.
00000100:00000001:2.0:1788575938.966384:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575938.966387:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575938.966389:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575938.966411:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575938.966416:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575938/real 1788575938]  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575938.966424:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575938.966426:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575938.966428:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575938.966430:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575938.966431:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575938.966433:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575938.966435:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575938.966440:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966448:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575938.966450:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966452:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575938.966453:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575938.966456:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966461:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575938.966465:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966469:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575938.966471:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966472:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575938.966472:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966474:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575938.966476:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575938.966478:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575938.966480:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575938.966481:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575938.966484:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575938.966488:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393246976:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575938.966493:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966495:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575938.966498:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393246976/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966504:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575938.966508:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575938.966511:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575938.966515:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575938.966517:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575938.966520:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575938.966522:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575938.966524:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:2.0:1788575938.966528:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575938.966531:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575938.966533:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:2.0:1788575938.966536:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575938.966537:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966540:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966548:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575938.966550:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575938.966551:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575938.966553:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575938.966554:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575938.966558:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966564:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575938.966565:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575938.966567:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575938.966569:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575938.966571:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575938.966574:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247104:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575938.966577:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575938.966579:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575938.966580:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575938.966582:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575938.966585:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575938.966586:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575938.966589:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575938.966591:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575938.966594:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575938.966596:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393247104, portal 10
00000100:00000001:2.0:1788575938.966598:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575938.966599:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00000040:2.0:1788575938.966604:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966608:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575938.966610:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575938.966611:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575938.966612:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393247104, offset 0
00000400:00000200:2.0:1788575938.966615:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575938.966617:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575938.966619:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575938.966621:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575938.966623:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575938.966624:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575938.966626:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575938.966628:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331186:331186:256:4294967295] 192.168.202.110@tcp LPNI seq info [331143:331143:8:4294967295]
00000400:00000200:2.0:1788575938.966631:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575938.966635:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575938.966638:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575938.966640:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575938.966642:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575938.966644:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575938.966647:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575938.966649:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575938.966651:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575938.966653:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575938.966654:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575938.966657:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575938.966659:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575938.966660:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575938.966661:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575938.966663:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966672:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575938.966674:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966678:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575938.966679:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575938.966680:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575938.966683:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575938.966684:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575938.966686:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307542567
00000100:00000001:2.0:1788575938.966688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307543481
00000100:00000001:2.0:1788575938.966689:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307542567
00000100:00000001:2.0:1788575938.966690:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307543481
00000100:00000001:2.0:1788575938.966691:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575938.966692:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575938.966693:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575938.966695:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575938/real 1788575938]  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575938.966700:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575938.966700:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575938.966701:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575938.966703:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575938.966704:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575938.966705:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575938.966706:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575938.966711:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966714:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575938.966715:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966717:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575938.966718:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575938.966720:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966723:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575938.966725:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966729:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575938.966730:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575938.966731:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966733:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575938.966734:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575938.966735:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575938.966737:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575938.966739:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575938.966741:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575938.966745:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247104:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575938.966747:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575938.966748:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966752:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575938.966753:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575938.966754:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575938.966755:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575938.966757:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575938.966760:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247232:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575938.966763:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575938.966765:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575938.966767:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575938.966773:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575938.966777:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575938.966779:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575938.966782:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575938.966785:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575938.966788:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575938.966791:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393247232, portal 4
00000100:00000001:2.0:1788575938.966793:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575938.966794:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:2.0:1788575938.966800:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966806:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575938.966808:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575938.966809:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575938.966811:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393247232, offset 0
00000400:00000200:2.0:1788575938.966814:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575938.966816:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575938.966819:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575938.966821:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575938.966823:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575938.966824:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575938.966826:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575938.966828:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331187:331187:256:4294967295] 192.168.202.110@tcp LPNI seq info [331144:331144:8:4294967295]
00000400:00000200:2.0:1788575938.966830:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575938.966834:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575938.966837:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575938.966839:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575938.966842:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575938.966844:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788575938.966846:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575938.966848:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788575938.966851:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575938.966853:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575938.966854:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575938.966857:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575938.966858:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575938.966859:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575938.966861:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575938.966863:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966868:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575938.966870:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966874:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575938.966875:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575938.966877:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575938.966880:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575938.966883:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575938.966884:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307542713
00000100:00000001:2.0:1788575938.966887:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307543481
00000100:00000001:2.0:1788575938.966889:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307542713
00000100:00000001:2.0:1788575938.966891:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307543481
00000100:00000001:2.0:1788575938.966893:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575938.966895:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575938.966896:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575938.966900:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575938/real 1788575938]  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575938.966908:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575938.966910:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575938.966912:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575938.966914:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575938.966916:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575938.966918:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575938.966921:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575938.966929:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966935:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575938.966937:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966938:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575938.966940:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575938.966943:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966949:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575938.966953:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966960:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575938.966962:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966964:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575938.966965:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966967:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575938.966969:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575938.966970:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575938.966973:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575938.966974:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575938.966977:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575938.966986:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247232:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575938.966989:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.966990:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575938.966992:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393247104/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.966995:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575938.966996:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575938.966998:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575938.967000:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575938.967001:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575938.967002:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575938.967003:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575938.967004:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:2.0:1788575938.967006:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575938.967007:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575938.967009:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:2.0:1788575938.967011:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575938.967013:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.967014:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575938.967017:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393247232/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575938.967023:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575938.967024:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575938.967025:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575938.967028:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575938.967029:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575938.967030:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575938.967032:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575938.967034:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:2.0:1788575938.967035:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575938.967037:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575938.967038:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:2.0:1788575938.967041:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575938.967043:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.967045:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575938.967048:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575938.967049:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575938.967051:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575938.967053:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575939.350125:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575939.350137:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575939.350139:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575939.350143:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575939.350149:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575939.350151:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575939.350154:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575939.350156:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575939.350160:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575939.350162:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575939.350166:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575939.350168:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575939.350231:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575939.350235:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575939.350239:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575939.477539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575939.477547:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575939.477549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575939.477555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575939.477559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575939.477564:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575939.477567:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575939.477569:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575939.477573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575939.925534:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307543481, at 4307543488
00000100:00000040:1.0:1788575939.925534:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307543481, at 4307543488
00000100:00000010:1.0:1788575939.925553:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00000100:00000010:0.0:1788575939.925556:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00010000:00000001:3.0:1788575940.373558:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575940.373562:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575940.373571:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575940.373572:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575940.373575:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575940.373575:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575940.373671:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575940.373678:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575940.373683:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575940.501525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575940.501534:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575940.501536:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575940.501541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575940.501544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575940.501548:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575940.501550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575940.501552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575940.501555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575941.397489:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575941.397494:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575941.397498:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575941.397499:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575941.397501:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575941.397502:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575941.397556:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575941.397560:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575941.397563:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575941.525526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575941.525533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575941.525535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575941.525541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575941.525545:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575941.525549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575941.525551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575941.525553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575941.525555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1788575942.037526:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1788575942.037534:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1788575942.037540:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:1.0:1788575942.037701:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:0.0:1788575942.037708:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788575942.037715:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788575942.037720:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788575942.037722:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788575942.421518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575942.421524:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575942.421529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575942.421530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575942.421533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575942.421535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575942.421597:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575942.421601:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575942.421611:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575942.549481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575942.549490:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575942.549492:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575942.549498:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575942.549502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575942.549506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575942.549509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575942.549510:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575942.549513:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575943.445505:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575943.445511:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575943.445515:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575943.445516:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575943.445518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575943.445519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575943.445573:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575943.445576:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575943.445578:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575943.573555:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575943.573563:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575943.573565:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575943.573570:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575943.573573:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575943.573576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575943.573579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575943.573581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575943.573583:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575944.085500:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575944.085518:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575944.085521:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575944.085525:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575944.085530:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575944.085537:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12875
00000400:00000200:3.0:1788575944.085547:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575944.085553:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575944.085555:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575944.085559:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085561:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575944.085563:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085564:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575944.085567:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575944.085569:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575944.085571:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575944.085573:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575944.085574:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575944.085576:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575944.085578:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085579:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575944.085580:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085581:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575944.085583:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575944.085586:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085587:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575944.085589:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575944.085591:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575944.085593:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575944.085594:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575944.085596:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575944.085604:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000020:00000040:3.0:1788575944.085607:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575944.085610:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575944.085612:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575944.085615:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575944.085617:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575944.085620:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575944.085638:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788575944.085645:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575944.085654:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575944.085662:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575944.085665:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575944.085668:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575944.085688:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575944.085690:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575944.085692:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575944.085693:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575944.085695:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575944.085696:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575944.085698:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12875
00000400:00000200:3.0:1788575944.085704:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575944.085707:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575944.085709:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575944.085712:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085715:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575944.085717:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085719:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575944.085721:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575944.085723:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575944.085725:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575944.085727:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575944.085730:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575944.085734:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575944.085744:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085747:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575944.085749:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085751:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575944.085753:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1788575944.085754:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575944.085760:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575944.085763:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788575944.085765:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575944.085769:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1788575944.085770:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575944.085779:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575944.085785:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575944.085786:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575944.085789:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575944.085793:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575944.085804:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575944.085812:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575944.085822:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575944.085824:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:3.0:1788575944.085826:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575944.085829:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575944.085830:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575944.085832:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575944.085836:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575944.085838:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788575944.085839:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575944.085845:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575944.085846:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575944.085860:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575944.085867:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393247360:192.168.202.110@tcp:250:kworker.0
02000000:00000010:3.0:1788575944.085869:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:2.0:1788575944.085878:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575944.085879:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575944.085885:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:3.0:1788575944.085886:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575944.085890:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575944.085899:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788575944.085901:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575944.085905:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575944.085907:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788575944.085910:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575944.085921:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575944.085923:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575944.085926:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575944.085929:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575944.085933:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1788575944.085934:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788575944.085935:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575944.085941:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575944.085944:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12875
00000400:00000200:3.0:1788575944.085948:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575944.085950:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:2.0:1788575944.085951:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00080000:3.0:1788575944.085952:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575944.085961:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.085962:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575944.085963:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:2.0:1788575944.085965:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000001:3.0:1788575944.085965:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575944.085970:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575944.085972:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575944.085973:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575944.085974:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575944.085978:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000200:2.0:1788575944.085979:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393247360, portal 25
00000100:00000001:2.0:1788575944.085992:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000020:00000001:3.0:1788575944.085992:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788575944.085999:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:3.0:1788575944.086001:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.086011:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575944.086013:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575944.086014:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575944.086020:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000040:2.0:1788575944.086021:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575944.086035:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575944.086047:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788575944.086051:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575944.086056:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575944.086058:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:2.0:1788575944.086062:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
02000000:00000001:3.0:1788575944.086064:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:2.0:1788575944.086069:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393247360, offset 0
02000000:00000001:3.0:1788575944.086069:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575944.086079:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575944.086082:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575944.086085:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00000001:3.0:1788575944.086091:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575944.086095:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575944.086099:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000400:00000200:2.0:1788575944.086101:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1788575944.086104:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575944.086110:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575944.086112:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575944.086116:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788575944.086117:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575944.086123:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575944.086125:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1788575944.086126:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575944.086134:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000010:3.0:1788575944.086135:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000400:00000200:2.0:1788575944.086142:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575944.086143:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575944.086148:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575944.086161:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331188:331188:256:4294967295] 192.168.202.110@tcp LPNI seq info [331145:331145:8:4294967295]
00000100:00100000:3.0:1788575944.086162:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575944.086181:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575944.086191:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575944.086198:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575944.086208:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1788575944.086214:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788575944.086223:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575944.086235:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000100:00000001:3.0:1788575944.086237:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575944.086242:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1788575944.086251:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788575944.086252:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:2.0:1788575944.086262:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:3.0:1788575944.086263:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12885)
00000800:00000200:2.1:1788575944.086279:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575944.086283:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575944.086288:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575944.086290:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575944.086306:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575944.086311:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575944.086316:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575944.086319:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575944.086323:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575944.086328:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086345:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575944.086349:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086357:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575944.086358:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575944.086364:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575944.086369:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575944.086372:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307548601
00000100:00000010:2.0:1788575944.086378:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c1e0.
00000100:00000001:2.0:1788575944.086386:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575944.086442:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307548602
00000100:00000010:2.0:1788575944.086446:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c7e0.
00000100:00000001:2.0:1788575944.086450:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575944.086454:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575944.086457:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575944.086460:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575944.086465:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575944/real 1788575944]  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575944.086488:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575944.086490:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575944.086493:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575944.086496:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575944.086498:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575944.086500:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575944.086504:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575944.086513:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086520:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575944.086523:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.086525:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575944.086527:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575944.086532:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086540:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575944.086544:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086552:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575944.086554:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.086556:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575944.086557:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.086560:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575944.086562:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575944.086565:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575944.086576:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575944.086581:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575944.086585:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575944.086593:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393247360:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575944.086598:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.086600:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575944.086604:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393247360/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086610:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575944.086613:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575944.086616:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575944.086619:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575944.086622:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575944.086624:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575944.086627:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575944.086630:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:2.0:1788575944.086634:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575944.086636:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575944.086638:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788575944.086642:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575944.086643:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.086646:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.086654:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575944.086656:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575944.086658:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575944.086660:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575944.086662:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575944.086666:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086673:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575944.086675:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575944.086677:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575944.086680:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575944.086682:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575944.086685:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575944.086688:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575944.086690:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575944.086692:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575944.086694:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575944.086697:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575944.086699:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575944.086702:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575944.086705:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575944.086709:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575944.086712:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393247488, portal 10
00000100:00000001:2.0:1788575944.086714:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575944.086715:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:2.0:1788575944.086722:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086730:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575944.086732:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575944.086734:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575944.086737:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393247488, offset 0
00000400:00000200:2.0:1788575944.086740:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575944.086744:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575944.086747:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575944.086750:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575944.086752:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575944.086754:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575944.086757:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575944.086760:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331189:331189:256:4294967295] 192.168.202.110@tcp LPNI seq info [331146:331146:8:4294967295]
00000400:00000200:2.0:1788575944.086764:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575944.086771:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575944.086775:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575944.086778:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575944.086787:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575944.086790:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575944.086794:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575944.086796:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575944.086800:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575944.086802:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575944.086805:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575944.086808:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575944.086811:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575944.086813:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575944.086815:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575944.086819:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086827:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575944.086830:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086844:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575944.086845:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575944.086848:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575944.086852:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575944.086856:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575944.086858:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307547687
00000100:00000001:2.0:1788575944.086860:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307548601
00000100:00000001:2.0:1788575944.086863:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307547687
00000100:00000001:2.0:1788575944.086865:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307548602
00000100:00000001:2.0:1788575944.086867:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575944.086869:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575944.086871:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575944.086874:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575944/real 1788575944]  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575944.086881:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575944.086882:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575944.086884:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575944.086886:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575944.086888:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575944.086890:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575944.086893:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575944.086901:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086907:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575944.086908:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.086910:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575944.086911:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575944.086915:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086922:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575944.086926:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086933:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575944.086935:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.086937:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575944.086938:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.086940:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575944.086942:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575944.086944:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575944.086947:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575944.086949:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575944.086952:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575944.086960:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575944.086964:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575944.086967:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.086974:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575944.086975:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575944.086978:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575944.086979:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575944.086982:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575944.086985:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575944.086988:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575944.086990:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575944.086993:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575944.086994:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575944.086997:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575944.086999:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575944.087003:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575944.087005:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575944.087008:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575944.087011:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393247616, portal 4
00000100:00000001:2.0:1788575944.087013:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575944.087014:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788575944.087020:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.087050:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575944.087052:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575944.087055:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575944.087057:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393247616, offset 0
00000400:00000200:2.0:1788575944.087061:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575944.087065:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575944.087068:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575944.087071:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575944.087073:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575944.087076:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575944.087078:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575944.087081:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331190:331190:256:4294967295] 192.168.202.110@tcp LPNI seq info [331147:331147:8:4294967295]
00000400:00000200:2.0:1788575944.087085:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575944.087090:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575944.087095:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575944.087098:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575944.087102:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575944.087104:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788575944.087108:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575944.087111:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788575944.087115:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575944.087117:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575944.087120:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575944.087123:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575944.087126:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575944.087127:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575944.087129:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575944.087133:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575969 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.087141:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575944.087144:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575969 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.087151:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575944.087152:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575944.087154:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575944.087157:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575944.087160:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575944.087162:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307547833
00000100:00000001:2.0:1788575944.087165:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307548601
00000100:00000001:2.0:1788575944.087167:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307547833
00000100:00000001:2.0:1788575944.087169:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307548602
00000100:00000001:2.0:1788575944.087170:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575944.087172:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575944.087174:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575944.087177:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575944/real 1788575944]  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575944.087185:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575944.087186:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575944.087188:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575944.087190:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575944.087191:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575944.087194:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575944.087197:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575944.087205:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.087211:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575944.087212:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.087214:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575944.087215:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575944.087218:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.087224:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575944.087228:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.087234:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575944.087237:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.087238:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575944.087240:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.087241:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575944.087243:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575944.087245:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575944.087248:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575944.087250:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575944.087254:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575944.087262:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575944.087265:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.087267:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575944.087270:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393247488/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.087277:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575944.087279:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575944.087281:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575944.087283:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575944.087284:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575944.087285:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575944.087288:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575944.087289:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:2.0:1788575944.087292:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575944.087293:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575944.087295:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788575944.087297:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575944.087299:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.087300:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575944.087303:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393247616/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575969 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575944.087310:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575944.087311:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575944.087312:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575944.087314:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575944.087315:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575944.087316:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575944.087318:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575944.087319:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:2.0:1788575944.087321:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575944.087322:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575944.087323:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788575944.087326:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575944.087327:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.087328:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.087331:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575944.087332:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575944.087334:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575944.087336:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575944.469532:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575944.469539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575944.469545:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575944.469546:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575944.469549:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575944.469550:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575944.469616:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575944.469620:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575944.469623:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575944.597541:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575944.597551:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575944.597553:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575944.597560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575944.597564:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575944.597569:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.597573:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575944.597575:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575944.597579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575945.045530:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307548601, at 4307548608
00000100:00000040:1.0:1788575945.045530:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307548602, at 4307548608
00000100:00000010:1.0:1788575945.045547:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00000100:00000010:0.0:1788575945.045550:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00010000:00000001:3.0:1788575945.493510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575945.493517:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575945.493522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575945.493523:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575945.493526:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575945.493528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575945.493582:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575945.493587:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575945.493591:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575945.621533:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575945.621544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575945.621547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575945.621555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575945.621562:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575945.621568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575945.621571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575945.621574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575945.621580:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575946.517500:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575946.517508:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575946.517513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575946.517520:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575946.517522:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575946.517525:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575946.517528:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575946.517530:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575946.517532:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575946.517534:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575946.517537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575946.517539:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575946.517540:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575946.517541:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575946.517543:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575946.517544:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575946.517545:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575946.517547:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575946.517606:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575946.517609:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575946.517613:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575946.648662:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575946.648670:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575946.648672:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575946.648678:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575946.648681:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575946.648685:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575946.648687:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575946.648690:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575946.648693:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575947.541592:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575947.541598:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575947.541605:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575947.541607:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575947.541610:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575947.541612:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575947.541716:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575947.541721:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575947.541724:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575947.669493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575947.669498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575947.669500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575947.669503:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575947.669506:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575947.669509:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575947.669511:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575947.669512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575947.669514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575948.565492:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575948.565497:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575948.565501:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575948.565502:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575948.565504:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575948.565505:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575948.565550:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575948.565554:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575948.565556:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575948.693502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575948.693509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575948.693512:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575948.693518:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575948.693522:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575948.693526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575948.693528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575948.693531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575948.693533:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575949.205607:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575949.205632:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575949.205634:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575949.205639:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575949.205644:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575949.205661:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12880
00000400:00000200:0.0:1788575949.205670:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575949.205677:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575949.205680:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575949.205683:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575949.205686:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575949.205689:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575949.205691:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575949.205695:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575949.205698:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575949.205701:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575949.205704:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575949.205706:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575949.205708:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575949.205711:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575949.205713:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575949.205715:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575949.205717:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575949.205720:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575949.205724:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575949.205726:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575949.205729:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575949.205732:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575949.205735:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575949.205737:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575949.205740:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575949.205750:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
00000020:00000040:0.0:1788575949.205753:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575949.205757:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575949.205759:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575949.205761:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575949.205764:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575949.205767:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575949.205774:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788575949.205780:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575949.205787:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575949.205796:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575949.205801:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575949.205805:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575949.205830:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575949.205833:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575949.205837:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575949.205838:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575949.205839:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575949.205842:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575949.205845:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12880
00000400:00000200:0.0:1788575949.205850:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575949.205853:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575949.205856:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575949.205857:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1788575949.205861:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575949.205861:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575949.205865:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575949.205866:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575949.205871:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575949.205874:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575949.205884:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575949.205887:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1788575949.205891:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575949.205891:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575949.205895:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
02000000:00000001:2.0:1788575949.205897:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:0.0:1788575949.205898:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1788575949.205901:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1788575949.205902:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575949.205908:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575949.205908:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575949.205913:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00100000:2.0:1788575949.205915:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393247744:192.168.202.110@tcp:250:kworker.0
00000100:00000040:0.0:1788575949.205918:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575949.205923:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575949.205924:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:0.0:1788575949.205926:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575949.205929:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575949.205929:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:0.0:1788575949.205931:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:2.0:1788575949.205933:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575949.205936:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575949.205939:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788575949.205940:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575949.205943:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:2.0:1788575949.205946:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575949.205950:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575949.205956:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575949.205959:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:2.0:1788575949.205961:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
02000000:00000001:0.0:1788575949.205965:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575949.205968:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:2.0:1788575949.205969:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000008:00100000:0.0:1788575949.205972:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000200:2.0:1788575949.205997:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393247744, portal 25
00000100:00000001:2.0:1788575949.206000:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575949.206001:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523511296 : -130484186040320 : ffff895348713800)
00000100:00000040:2.0:1788575949.206006:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206011:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575949.206014:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575949.206017:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575949.206020:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393247744, offset 0
00000400:00000200:2.0:1788575949.206025:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575949.206031:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575949.206036:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575949.206041:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00000001:0.0:1788575949.206042:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575949.206045:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575949.206049:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575949.206051:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575949.206053:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331191:331191:256:4294967295] 192.168.202.110@tcp LPNI seq info [331148:331148:8:4294967295]
00000100:00000001:0.0:1788575949.206056:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575949.206057:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575949.206064:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000010:0.0:1788575949.206072:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000800:00000200:2.0:1788575949.206072:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575949.206076:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000020:00000040:0.0:1788575949.206078:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575949.206082:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575949.206085:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575949.206086:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575949.206088:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:2.0:1788575949.206093:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575949.206100:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575949.206103:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
02000000:00000001:0.0:1788575949.206107:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000100:2.0:1788575949.206107:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575949.206110:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
02000000:00000001:0.0:1788575949.206111:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575949.206117:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:0.0:1788575949.206119:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000400:00000200:2.0:1788575949.206124:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1788575949.206126:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575949.206131:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000100:00100000:0.0:1788575949.206134:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1788575949.206135:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575949.206140:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575949.206146:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575949.206149:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575949.206154:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.206161:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575949.206162:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575949.206168:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575949.206181:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.206183:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575949.206185:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575949.206187:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00080000:0.0:1788575949.206191:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575949.206194:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575949.206196:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575949.206196:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575949.206201:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788575949.206204:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788575949.206207:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12880
00000100:00000001:2.0:1788575949.206210:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307553721
00000400:00000200:0.0:1788575949.206215:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1788575949.206218:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c300.
00000400:00000200:0.0:1788575949.206220:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575949.206224:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575949.206226:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:0.0:1788575949.206229:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575949.206233:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575949.206234:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307553721
00000100:00000040:0.0:1788575949.206237:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000010:2.0:1788575949.206239:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c2a0.
00000100:00000001:0.0:1788575949.206241:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575949.206244:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000001:0.0:1788575949.206245:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575949.206249:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:2.0:1788575949.206249:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1788575949.206252:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575949.206253:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575949.206256:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:2.0:1788575949.206257:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000020:00000001:0.0:1788575949.206261:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00100000:2.0:1788575949.206265:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575949/real 1788575949]  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1788575949.206266:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575949.206273:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575949.206278:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575949.206282:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575949.206285:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000001:2.0:1788575949.206285:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:0.0:1788575949.206287:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575949.206290:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575949.206292:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575949.206294:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575949.206297:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000001:2.0:1788575949.206299:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:0.0:1788575949.206300:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575949.206303:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000100:00000001:0.0:1788575949.206304:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575949.206308:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575949.206309:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000008:00000001:0.0:1788575949.206313:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575949.206313:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:0.0:1788575949.206317:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.206319:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000010:0.0:1788575949.206323:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000100:00000200:2.0:1788575949.206325:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575949.206327:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575949.206332:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575949.206334:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575949.206338:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575949.206340:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575949.206343:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575949.206343:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788575949.206350:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:2.0:1788575949.206353:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575949.206355:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.206356:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.206360:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00100000:0.0:1788575949.206363:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575949.206364:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575949.206373:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575949.206376:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575949.206380:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.206386:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575949.206388:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575949.206416:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575949.206424:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575949.206429:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575949.206430:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575949.206438:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12890)
00000100:00000001:2.0:1788575949.206441:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575949.206446:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.206448:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575949.206449:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.206452:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575949.206454:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575949.206457:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575949.206461:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575949.206464:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575949.206476:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575949.206484:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393247744:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575949.206489:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.206492:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575949.206494:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713800 x1875444393247744/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206506:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575949.206508:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575949.206511:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575949.206514:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575949.206516:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575949.206517:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575949.206520:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575949.206523:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:2.0:1788575949.206526:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575949.206527:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575949.206529:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713800.
00000100:00000001:2.0:1788575949.206532:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575949.206533:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.206535:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.206543:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575949.206544:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.206547:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575949.206548:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575949.206550:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575949.206554:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206560:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575949.206561:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575949.206563:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575949.206565:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575949.206566:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575949.206569:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247872:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575949.206571:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575949.206573:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575949.206574:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575949.206576:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575949.206579:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575949.206581:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575949.206585:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575949.206588:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575949.206591:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575949.206594:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393247872, portal 10
00000100:00000001:2.0:1788575949.206596:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575949.206597:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523503232 : -130484186048384 : ffff895348711880)
00000100:00000040:2.0:1788575949.206603:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206609:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575949.206611:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575949.206613:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575949.206615:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393247872, offset 0
00000400:00000200:2.0:1788575949.206619:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575949.206622:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575949.206666:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575949.206669:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575949.206673:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575949.206675:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575949.206677:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575949.206679:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331192:331192:256:4294967295] 192.168.202.110@tcp LPNI seq info [331149:331149:8:4294967295]
00000400:00000200:2.0:1788575949.206683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575949.206689:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575949.206693:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575949.206696:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575949.206700:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575949.206703:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575949.206714:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575949.206717:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575949.206721:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575949.206722:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575949.206725:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575949.206728:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575949.206732:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575949.206734:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575949.206736:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575949.206745:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206752:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575949.206755:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206761:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.206763:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575949.206766:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575949.206769:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575949.206773:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575949.206774:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307552807
00000100:00000001:2.0:1788575949.206778:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307553721
00000100:00000001:2.0:1788575949.206780:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307552807
00000100:00000001:2.0:1788575949.206782:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307553721
00000100:00000001:2.0:1788575949.206784:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575949.206786:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.206788:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575949.206792:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575949/real 1788575949]  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575949.206800:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575949.206801:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575949.206803:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575949.206805:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575949.206807:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575949.206809:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575949.206812:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575949.206818:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206824:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575949.206825:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.206827:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575949.206828:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575949.206832:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206838:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575949.206842:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206848:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575949.206851:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.206853:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575949.206854:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.206856:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575949.206858:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575949.206860:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575949.206863:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575949.206865:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575949.206868:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575949.206876:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393247872:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575949.206880:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575949.206883:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206889:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575949.206891:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575949.206892:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575949.206894:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575949.206895:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575949.206898:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248000:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575949.206901:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575949.206902:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575949.206904:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575949.206906:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575949.206909:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575949.206911:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575949.206914:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575949.206917:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575949.206920:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575949.206922:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393248000, portal 4
00000100:00000001:2.0:1788575949.206924:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575949.206926:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:2.0:1788575949.206931:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.206937:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575949.206939:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575949.206941:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575949.206944:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393248000, offset 0
00000400:00000200:2.0:1788575949.206948:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575949.206951:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575949.206954:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575949.206957:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575949.206959:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575949.206961:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575949.206963:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575949.206966:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331193:331193:256:4294967295] 192.168.202.110@tcp LPNI seq info [331150:331150:8:4294967295]
00000400:00000200:2.0:1788575949.206970:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575949.206986:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575949.206991:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575949.206994:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575949.206997:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575949.207000:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788575949.207003:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575949.207006:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788575949.207009:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575949.207011:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575949.207014:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575949.207017:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575949.207019:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575949.207021:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575949.207023:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575949.207026:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575974 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.207032:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575949.207035:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575974 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.207041:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.207042:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575949.207045:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575949.207048:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575949.207051:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575949.207053:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307552953
00000100:00000001:2.0:1788575949.207056:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307553721
00000100:00000001:2.0:1788575949.207057:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307552953
00000100:00000001:2.0:1788575949.207059:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307553721
00000100:00000001:2.0:1788575949.207061:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575949.207063:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.207065:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575949.207068:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575949/real 1788575949]  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575949.207075:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575949.207076:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575949.207078:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575949.207080:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575949.207082:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575949.207083:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575949.207087:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575949.207094:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.207100:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575949.207101:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.207103:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575949.207104:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575949.207107:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.207114:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575949.207117:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.207123:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575949.207126:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.207127:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575949.207128:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.207130:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575949.207131:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575949.207133:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575949.207136:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575949.207138:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575949.207142:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575949.207149:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248000:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575949.207152:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.207153:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575949.207156:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711880 x1875444393247872/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.207162:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575949.207163:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575949.207165:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575949.207167:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575949.207168:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575949.207169:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575949.207171:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575949.207173:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788575949.207175:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575949.207176:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575949.207178:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711880.
00000100:00000001:2.0:1788575949.207180:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575949.207181:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.207182:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575949.207185:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393248000/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575974 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575949.207191:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575949.207192:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575949.207193:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575949.207195:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575949.207196:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575949.207197:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575949.207198:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575949.207199:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788575949.207201:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575949.207202:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575949.207204:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:2.0:1788575949.207205:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575949.207206:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.207208:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.207210:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575949.207212:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575949.207213:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575949.207215:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575949.589509:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575949.589517:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575949.589519:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575949.589522:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575949.589527:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575949.589529:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575949.589531:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575949.589533:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575949.589536:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575949.589538:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575949.589542:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575949.589543:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575949.589647:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575949.589650:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575949.589653:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575949.717510:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575949.717517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575949.717519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575949.717524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575949.717528:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575949.717531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.717534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575949.717537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575949.717539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575950.165502:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307553721, at 4307553728
00000100:00000040:1.0:1788575950.165502:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307553721, at 4307553728
00000100:00000010:0.0:1788575950.165517:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00000100:00000010:1.0:1788575950.165521:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00010000:00000001:3.0:1788575950.613473:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575950.613478:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575950.613482:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575950.613483:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575950.613485:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575950.613486:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575950.613529:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575950.613533:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575950.613537:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575950.741460:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575950.741467:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575950.741468:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575950.741472:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575950.741475:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575950.741478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575950.741479:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575950.741481:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575950.741483:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575951.637659:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575951.637735:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575951.637745:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575951.637747:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575951.637750:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575951.637751:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575951.638207:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575951.638211:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575951.638215:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575951.765518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575951.765525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575951.765527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575951.765531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575951.765534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575951.765538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575951.765540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575951.765541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575951.765543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575952.661514:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575952.661520:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575952.661525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575952.661526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575952.661530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575952.661531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575952.661593:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575952.661597:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575952.661599:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575952.789511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575952.789518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575952.789521:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575952.789526:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575952.789530:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575952.789534:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575952.789537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575952.789539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575952.789542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575953.685481:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575953.685487:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575953.685494:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575953.685495:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575953.685499:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575953.685500:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575953.685545:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575953.685548:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575953.685551:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575953.813543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575953.813551:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575953.813553:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575953.813559:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575953.813563:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575953.813567:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575953.813570:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575953.813572:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575953.813576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575954.325504:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575954.325522:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575954.325525:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575954.325530:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575954.325534:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575954.325542:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12885
00000400:00000200:0.0:1788575954.325552:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575954.325559:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575954.325561:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575954.325565:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325569:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575954.325571:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325574:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575954.325578:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575954.325580:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575954.325583:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575954.325587:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575954.325589:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575954.325592:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575954.325595:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325597:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575954.325599:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325600:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575954.325604:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575954.325607:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325609:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575954.325612:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575954.325615:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575954.325618:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575954.325620:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575954.325622:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575954.325631:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788575954.325635:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575954.325639:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575954.325641:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575954.325644:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575954.325646:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575954.325650:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575954.325657:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788575954.325664:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575954.325674:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575954.325688:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575954.325692:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575954.325696:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575954.325721:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575954.325725:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575954.325728:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575954.325729:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575954.325731:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575954.325734:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575954.325736:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12885
00000400:00000200:0.0:1788575954.325740:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575954.325743:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575954.325745:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575954.325748:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325750:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575954.325752:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325754:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575954.325755:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575954.325757:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575954.325758:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575954.325760:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575954.325762:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575954.325765:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575954.325768:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325770:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575954.325772:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325774:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575954.325776:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1788575954.325778:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788575954.325781:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1788575954.325783:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575954.325784:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788575954.325789:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575954.325793:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1788575954.325793:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575954.325798:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575954.325802:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1788575954.325803:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.325805:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575954.325809:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788575954.325810:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575954.325814:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:0.0:1788575954.325815:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575954.325816:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788575954.325822:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575954.325822:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393248128:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575954.325827:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1788575954.325828:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575954.325831:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:0.0:1788575954.325833:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
02000000:00000001:2.0:1788575954.325833:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575954.325837:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:0.0:1788575954.325839:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000010:2.0:1788575954.325842:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788575954.325843:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575954.325847:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575954.325848:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575954.325851:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575954.325854:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575954.325854:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575954.325859:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575954.325859:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575954.325867:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
02000000:00000010:0.0:1788575954.325869:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000200:2.0:1788575954.325873:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393248128, portal 25
00000100:00000001:0.0:1788575954.325875:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575954.325883:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575954.325889:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575954.325891:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575954.325894:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575954.325902:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.325904:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1788575954.325905:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575954.325908:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00080000:0.0:1788575954.325910:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575954.325914:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575954.325915:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575954.325920:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575954.325924:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12885
00000400:00000200:0.0:1788575954.325959:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575954.325962:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575954.325965:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575954.325967:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325969:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575954.325971:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575954.325972:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.325978:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575954.325982:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575954.325985:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575954.325989:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575954.325989:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:0.0:1788575954.325992:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000040:2.0:1788575954.325997:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:0.0:1788575954.325998:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575954.326002:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:2.0:1788575954.326003:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575954.326008:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393248128, offset 0
00000400:00000200:2.0:1788575954.326016:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788575954.326018:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.326021:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575954.326023:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575954.326026:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575954.326027:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1788575954.326030:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575954.326035:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575954.326038:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788575954.326040:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575954.326043:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575954.326046:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1788575954.326048:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575954.326052:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575954.326052:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1788575954.326056:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788575954.326057:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00100000:0.0:1788575954.326060:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575954.326061:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575954.326068:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331194:331194:256:4294967295] 192.168.202.110@tcp LPNI seq info [331151:331151:8:4294967295]
00000008:00000001:0.0:1788575954.326069:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575954.326074:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575954.326074:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1788575954.326079:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000400:00000200:2.0:1788575954.326087:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:0.0:1788575954.326093:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.0:1788575954.326094:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575954.326100:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:2.0:1788575954.326104:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
02000000:00000001:0.0:1788575954.326105:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575954.326109:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575954.326112:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575954.326115:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575954.326116:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575954.326122:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
02000000:00000010:0.0:1788575954.326124:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000800:00000200:2.1:1788575954.326128:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575954.326131:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788575954.326131:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00100000:0.0:1788575954.326140:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:2.0:1788575954.326148:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575954.326152:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000100:00000001:0.0:1788575954.326153:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575954.326157:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575954.326158:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1788575954.326163:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575954.326164:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575954.326171:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575954.326173:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:0.0:1788575954.326179:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.326180:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:0.0:1788575954.326182:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1788575954.326186:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575954.326188:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575954.326197:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12895)
00000100:00000001:2.0:1788575954.326198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575954.326205:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.326213:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.326215:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575954.326221:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575954.326225:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575954.326228:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307558841
00000100:00000010:2.0:1788575954.326234:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ca80.
00000100:00000001:2.0:1788575954.326241:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575954.327233:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307558842
00000100:00000010:2.0:1788575954.327239:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c6c0.
00000100:00000001:2.0:1788575954.327243:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575954.327247:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575954.327250:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.327252:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575954.327258:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575954/real 1788575954]  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575954.327266:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575954.327267:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575954.327268:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575954.327270:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575954.327272:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575954.327274:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575954.327276:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575954.327281:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327285:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575954.327286:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327288:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575954.327290:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575954.327292:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327296:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575954.327299:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327303:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575954.327305:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327306:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575954.327307:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327308:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575954.327310:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575954.327311:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575954.327314:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575954.327316:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575954.327319:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575954.327324:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393248128:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575954.327326:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327328:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575954.327329:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393248128/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327333:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575954.327335:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575954.327337:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575954.327339:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575954.327340:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575954.327341:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575954.327342:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575954.327344:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:2.0:1788575954.327347:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575954.327348:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575954.327349:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:2.0:1788575954.327351:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575954.327352:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327354:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327361:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575954.327362:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.327365:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575954.327366:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575954.327367:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575954.327371:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327377:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575954.327378:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575954.327380:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575954.327382:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575954.327385:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575954.327388:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248256:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575954.327406:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575954.327408:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575954.327409:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575954.327411:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575954.327414:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575954.327416:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575954.327420:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575954.327423:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575954.327427:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575954.327430:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393248256, portal 10
00000100:00000001:2.0:1788575954.327431:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575954.327433:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000040:2.0:1788575954.327438:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327442:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575954.327444:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575954.327445:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575954.327447:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393248256, offset 0
00000400:00000200:2.0:1788575954.327450:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575954.327453:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575954.327456:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575954.327458:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575954.327460:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575954.327461:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575954.327463:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575954.327465:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331195:331195:256:4294967295] 192.168.202.110@tcp LPNI seq info [331152:331152:8:4294967295]
00000400:00000200:2.0:1788575954.327468:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575954.327473:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575954.327478:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575954.327481:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575954.327485:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575954.327488:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575954.327493:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575954.327495:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575954.327499:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575954.327500:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575954.327503:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575954.327506:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575954.327509:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575954.327511:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575954.327513:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575954.327517:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327524:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575954.327527:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327533:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.327534:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575954.327537:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575954.327541:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575954.327545:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575954.327547:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307557928
00000100:00000001:2.0:1788575954.327550:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307558841
00000100:00000001:2.0:1788575954.327551:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307557928
00000100:00000001:2.0:1788575954.327553:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307558842
00000100:00000001:2.0:1788575954.327555:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575954.327557:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.327559:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575954.327563:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575954/real 1788575954]  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575954.327577:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575954.327578:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575954.327580:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575954.327583:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575954.327585:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575954.327587:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575954.327590:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575954.327598:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327605:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575954.327607:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575954.327610:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575954.327613:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327620:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575954.327624:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327630:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575954.327633:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327635:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575954.327636:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327637:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575954.327639:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575954.327643:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575954.327646:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575954.327648:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575954.327653:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575954.327660:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248256:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575954.327664:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575954.327667:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327673:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575954.327675:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575954.327677:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575954.327678:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575954.327680:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575954.327683:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575954.327687:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575954.327689:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575954.327691:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575954.327692:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575954.327696:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575954.327698:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575954.327702:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575954.327705:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575954.327708:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575954.327711:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393248384, portal 4
00000100:00000001:2.0:1788575954.327712:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575954.327714:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:2.0:1788575954.327720:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327725:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575954.327727:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575954.327730:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575954.327732:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393248384, offset 0
00000400:00000200:2.0:1788575954.327737:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575954.327740:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575954.327744:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575954.327747:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575954.327750:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575954.327752:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575954.327755:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575954.327758:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331196:331196:256:4294967295] 192.168.202.110@tcp LPNI seq info [331153:331153:8:4294967295]
00000400:00000200:2.0:1788575954.327761:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575954.327767:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575954.327771:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575954.327774:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575954.327777:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575954.327779:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788575954.327783:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575954.327786:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788575954.327789:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575954.327791:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575954.327794:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575954.327797:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575954.327799:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575954.327801:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575954.327803:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575954.327806:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575979 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327814:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575954.327816:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575979 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327822:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.327823:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575954.327826:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575954.327829:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575954.327831:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575954.327834:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307558074
00000100:00000001:2.0:1788575954.327836:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307558841
00000100:00000001:2.0:1788575954.327838:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307558074
00000100:00000001:2.0:1788575954.327840:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307558842
00000100:00000001:2.0:1788575954.327842:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575954.327844:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.327846:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575954.327849:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575954/real 1788575954]  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575954.327856:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575954.327858:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575954.327859:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575954.327861:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575954.327863:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575954.327865:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575954.327868:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575954.327876:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327882:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575954.327884:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327886:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575954.327887:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575954.327890:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327896:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575954.327900:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327906:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575954.327908:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327910:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575954.327911:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327912:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575954.327914:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575954.327916:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575954.327919:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575954.327922:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575954.327926:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575954.327946:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575954.327950:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327952:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575954.327955:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393248256/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327962:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575954.327964:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575954.327966:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575954.327969:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575954.327970:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575954.327971:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575954.327973:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575954.327975:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:2.0:1788575954.327977:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575954.327978:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575954.327979:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:2.0:1788575954.327981:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575954.327982:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.327984:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575954.327986:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393248384/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575979 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575954.327996:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575954.327998:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575954.327999:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788575954.328001:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575954.328002:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575954.328003:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575954.328005:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575954.328006:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788575954.328008:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575954.328009:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575954.328010:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:2.0:1788575954.328012:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575954.328013:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.328015:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.328018:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575954.328019:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575954.328020:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575954.328022:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575954.709766:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575954.709771:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575954.709775:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575954.709776:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575954.709778:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575954.709779:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575954.712189:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575954.712193:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575954.712197:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575954.837559:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575954.837566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575954.837568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575954.837573:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575954.837577:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575954.837593:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.837595:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575954.837598:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575954.837601:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575955.285516:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307558841, at 4307558848
00000100:00000040:1.0:1788575955.285521:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307558842, at 4307558848
00000100:00000010:0.0:1788575955.285532:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000100:00000010:1.0:1788575955.285535:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00010000:00000001:3.0:1788575955.733527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575955.733535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575955.733540:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575955.733542:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575955.733545:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575955.733547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575955.733611:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575955.733616:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575955.733619:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575955.861634:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575955.861642:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575955.861644:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575955.861649:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575955.861652:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575955.861656:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575955.861659:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575955.861661:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575955.861664:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575956.757510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575956.757518:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575956.757523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575956.757526:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575956.757528:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575956.757532:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575956.757535:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575956.757537:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575956.757539:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575956.757541:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575956.757545:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575956.757546:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575956.757547:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575956.757548:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575956.757549:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575956.757551:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575956.757552:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575956.757554:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575956.757613:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575956.757617:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575956.757620:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575956.885486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575956.885493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575956.885495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575956.885501:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575956.885504:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575956.885508:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575956.885510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575956.885511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575956.885514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575957.781947:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575957.781953:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575957.781958:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575957.781960:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575957.781963:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575957.781964:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575957.782137:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575957.782140:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575957.782143:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575957.909486:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575957.909493:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575957.909495:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575957.909500:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575957.909503:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575957.909506:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575957.909509:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575957.909511:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575957.909514:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575958.805736:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575958.805747:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575958.805750:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575958.805753:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575958.806030:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575958.806033:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575958.806035:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575958.806038:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575958.806043:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575958.806044:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575958.806048:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575958.806050:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575958.806182:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575958.806187:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575958.806191:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575958.933543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575958.933553:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575958.933556:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575958.933563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575958.933567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575958.933572:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575958.933575:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575958.933578:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575958.933581:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575959.445717:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575959.445727:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575959.445729:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575959.445733:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575959.445737:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575959.445744:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12890
00000400:00000200:0.0:1788575959.445752:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575959.445759:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575959.445762:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575959.445766:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.445769:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575959.445771:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.445773:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575959.445777:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575959.445779:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575959.445782:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575959.445785:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575959.445787:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575959.445789:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575959.445791:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.445793:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575959.445795:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.445796:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575959.445799:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575959.445802:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575959.445805:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575959.445808:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575959.445811:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575959.445814:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575959.445816:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575959.445819:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575959.445828:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788575959.445832:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575959.445835:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575959.445837:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575959.445840:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575959.445842:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575959.445845:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575959.445852:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788575959.445860:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575959.445869:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575959.445984:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575959.445989:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575959.445994:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575959.446015:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575959.446018:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575959.446021:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575959.446023:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575959.446025:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575959.446028:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575959.446032:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12890
00000100:00000001:2.0:1788575959.446035:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1788575959.446038:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575959.446040:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575959.446043:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000400:00000200:0.0:1788575959.446044:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575959.446047:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575959.446049:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575959.446053:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575959.446059:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575959.446061:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575959.446063:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575959.446065:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575959.446067:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575959.446077:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393248512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575959.446080:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575959.446084:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575959.446086:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575959.446088:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575959.446093:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575959.446095:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575959.446100:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575959.446104:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575959.446110:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575959.446118:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393248512, portal 25
00000100:00000001:2.0:1788575959.446120:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575959.446122:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000040:2.0:1788575959.446128:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446133:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575959.446135:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575959.446137:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575959.446140:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393248512, offset 0
00000400:00000200:2.0:1788575959.446144:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575959.446150:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575959.446156:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575959.446158:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:0.0:1788575959.446163:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575959.446166:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575959.446168:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575959.446171:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:0.0:1788575959.446173:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575959.446176:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788575959.446178:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575959.446181:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788575959.446182:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575959.446184:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575959.446188:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446189:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575959.446192:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446193:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575959.446195:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788575959.446198:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446200:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575959.446202:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575959.446207:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575959.446210:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575959.446211:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575959.446215:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575959.446218:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575959.446220:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575959.446224:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788575959.446226:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575959.446228:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575959.446229:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575959.446231:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575959.446232:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575959.446234:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575959.446238:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575959.446242:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575959.446248:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575959.446255:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575959.446257:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575959.446261:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575959.446268:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575959.446271:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575959.446274:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575959.446275:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575959.446277:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575959.446280:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575959.446282:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12890
00000400:00000200:0.0:1788575959.446287:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575959.446290:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575959.446292:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575959.446294:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446296:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575959.446298:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446300:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575959.446302:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575959.446303:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575959.446305:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575959.446307:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575959.446309:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575959.446311:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575959.446314:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446315:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575959.446317:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446319:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575959.446321:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575959.446323:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446325:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575959.446327:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575959.446330:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575959.446332:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575959.446333:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575959.446336:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788575959.446338:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00000001:0.0:1788575959.446340:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575959.446344:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446345:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575959.446350:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788575959.446354:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575959.446355:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331197:331197:256:4294967295] 192.168.202.110@tcp LPNI seq info [331154:331154:8:4294967295]
02000000:00000001:0.0:1788575959.446356:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575959.446358:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575959.446362:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575959.446364:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575959.446367:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575959.446368:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575959.446374:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788575959.446379:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575959.446383:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00100000:0.0:1788575959.446388:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1788575959.446407:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575959.446411:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575959.446416:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575959.446419:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575959.446423:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575959.446425:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575959.446427:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575959.446429:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575959.446434:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575959.446438:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575959.446442:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575959.446444:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575959.446447:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575959.446450:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446456:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575959.446459:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446463:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575959.446464:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575959.446469:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575959.446471:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575959.446473:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307563961
00000100:00000010:2.0:1788575959.446477:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cc00.
00000100:00000001:2.0:1788575959.446482:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575959.446486:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307563961
00000100:00000010:2.0:1788575959.446488:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c3c0.
00000100:00000001:2.0:1788575959.446490:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575959.446492:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575959.446494:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575959.446496:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575959.446500:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575959/real 1788575959]  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575959.446506:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575959.446508:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575959.446510:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575959.446511:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575959.446512:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575959.446513:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575959.446515:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575959.446520:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446524:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575959.446525:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446527:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575959.446528:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575959.446531:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446535:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575959.446538:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446542:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575959.446544:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446546:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575959.446546:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446548:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575959.446549:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575959.446551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575959.446554:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575959.446555:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575959.446557:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575959.446562:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393248512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575959.446565:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446566:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575959.446568:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393248512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446572:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575959.446574:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575959.446576:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575959.446578:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575959.446579:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575959.446580:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575959.446581:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575959.446583:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:2.0:1788575959.446586:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575959.446587:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575959.446588:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:2.0:1788575959.446590:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575959.446591:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446592:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446598:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575959.446610:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575959.446612:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575959.446613:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575959.446615:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575959.446617:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446625:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575959.446626:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575959.446628:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575959.446629:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575959.446630:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575959.446632:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575959.446634:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575959.446635:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575959.446636:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575959.446638:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575959.446640:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575959.446641:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575959.446643:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575959.446645:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575959.446647:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575959.446649:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393248640, portal 10
00000100:00000001:2.0:1788575959.446650:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575959.446651:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
00000100:00000040:2.0:1788575959.446655:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446659:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575959.446660:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575959.446662:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575959.446663:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393248640, offset 0
00000400:00000200:2.0:1788575959.446665:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575959.446668:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575959.446670:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575959.446672:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575959.446674:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575959.446675:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575959.446676:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575959.446678:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331198:331198:256:4294967295] 192.168.202.110@tcp LPNI seq info [331155:331155:8:4294967295]
00000400:00000200:2.0:1788575959.446683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575959.446693:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788575959.446695:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:2.0:1788575959.446697:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575959.446700:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575959.446701:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575959.446705:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575959.446709:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:0.0:1788575959.446710:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788575959.446715:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575959.446719:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575959.446727:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575959.446731:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000100:00000001:0.0:1788575959.446733:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575959.446734:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1788575959.446737:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575959.446738:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575959.446742:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000100:00000040:0.0:1788575959.446745:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:2.0:1788575959.446745:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575959.446747:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575959.446751:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575959.446754:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12900)
00000100:00000001:2.0:1788575959.446758:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575959.446762:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446766:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575959.446767:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575959.446770:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575959.446772:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575959.446774:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575959.446776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307563047
00000100:00000001:2.0:1788575959.446778:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307563961
00000100:00000001:2.0:1788575959.446780:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307563047
00000100:00000001:2.0:1788575959.446781:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307563961
00000100:00000001:2.0:1788575959.446783:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575959.446785:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575959.446786:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575959.446790:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575959/real 1788575959]  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575959.446797:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575959.446799:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575959.446801:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575959.446803:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575959.446805:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575959.446807:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575959.446810:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575959.446818:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446824:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575959.446825:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446827:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575959.446829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575959.446831:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446837:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575959.446840:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446846:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575959.446848:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446850:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575959.446851:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446852:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575959.446854:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575959.446856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575959.446859:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575959.446861:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575959.446864:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575959.446872:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575959.446876:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446877:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575959.446880:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393248640/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446921:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575959.446923:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575959.446925:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575959.446928:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575959.446930:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575959.446931:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575959.446933:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575959.446935:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:2.0:1788575959.446936:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575959.446937:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575959.446939:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:2.0:1788575959.446941:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575959.446942:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446944:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.446947:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575959.446948:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575959.446950:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575959.446951:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575959.446953:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575959.446956:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.446963:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575959.446964:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575959.446966:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575959.446968:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575959.446970:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575959.446973:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248768:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575959.446976:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575959.446978:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575959.446979:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575959.446981:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575959.446983:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575959.446985:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575959.446989:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575959.446992:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575959.446995:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575959.446998:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393248768, portal 4
00000100:00000001:2.0:1788575959.446999:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575959.447000:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:2.0:1788575959.447005:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.447011:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575959.447013:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575959.447014:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575959.447016:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393248768, offset 0
00000400:00000200:2.0:1788575959.447019:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575959.447023:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575959.447026:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575959.447029:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575959.447031:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575959.447033:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575959.447035:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575959.447037:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331199:331199:256:4294967295] 192.168.202.110@tcp LPNI seq info [331156:331156:8:4294967295]
00000400:00000200:2.0:1788575959.447041:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575959.447046:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575959.447050:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575959.447053:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575959.447056:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575959.447059:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1788575959.447062:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575959.447064:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1788575959.447080:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575959.447081:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575959.447084:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575959.447087:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575959.447091:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575959.447093:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575959.447096:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575959.447099:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575984 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.447106:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575959.447109:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575984 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.447115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575959.447116:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575959.447119:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575959.447122:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575959.447125:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575959.447127:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307563193
00000100:00000001:2.0:1788575959.447129:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307563961
00000100:00000001:2.0:1788575959.447131:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307563193
00000100:00000001:2.0:1788575959.447133:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307563961
00000100:00000001:2.0:1788575959.447135:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575959.447137:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575959.447139:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575959.447142:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575959/real 1788575959]  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575959.447149:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575959.447151:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575959.447152:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575959.447155:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575959.447156:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575959.447158:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575959.447161:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575959.447170:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.447530:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575959.447532:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.447534:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575959.447536:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575959.447539:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.447544:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575959.447546:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.447550:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575959.447552:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.447553:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575959.447553:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.447554:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575959.447555:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575959.447557:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575959.447559:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575959.447560:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575959.447562:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575959.447566:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393248768:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575959.447569:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.447570:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575959.447572:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393248768/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575984 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575959.447575:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575959.447576:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575959.447578:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575959.447580:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575959.447581:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575959.447582:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575959.447584:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575959.447585:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:2.0:1788575959.447586:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575959.447587:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575959.447588:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:2.0:1788575959.447590:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575959.447590:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.447592:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575959.447594:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575959.447595:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575959.447596:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575959.447639:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575959.829748:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575959.829754:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575959.829760:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575959.829761:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575959.829765:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575959.829766:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575959.832006:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575959.832009:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575959.832012:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575959.957600:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575959.957605:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575959.957608:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575959.957613:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575959.957617:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575959.957620:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575959.957623:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575959.957625:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575959.957628:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575960.405480:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307563961, at 4307563968
00000100:00000040:1.0:1788575960.405480:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307563961, at 4307563968
00000100:00000010:1.0:1788575960.405496:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000100:00000010:0.0:1788575960.405498:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00010000:00000001:3.0:1788575960.854181:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575960.854187:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575960.854192:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575960.854193:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575960.854196:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575960.854198:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575960.854270:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575960.854275:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575960.854278:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575960.981540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575960.981549:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575960.981552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575960.981558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575960.981562:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575960.981567:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575960.981570:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575960.981573:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575960.981576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575961.877551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575961.877560:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575961.877575:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575961.877577:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575961.877581:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575961.877582:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575961.877776:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575961.877781:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575961.877785:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575962.005634:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575962.005643:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575962.005646:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575962.005652:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575962.005657:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575962.005662:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575962.005665:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575962.005668:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575962.005672:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575962.901511:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575962.901516:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575962.901520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575962.901520:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575962.901523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575962.901524:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575962.901588:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575962.901592:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575962.901595:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575963.029613:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575963.029621:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575963.029623:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575963.029627:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575963.029630:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575963.029633:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575963.029635:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575963.029636:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575963.029639:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575963.925508:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575963.925515:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575963.925520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575963.925521:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575963.925525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575963.925526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575963.925616:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575963.925619:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575963.925622:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575964.053498:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575964.053503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575964.053505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575964.053509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575964.053511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575964.053514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575964.053516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575964.053517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575964.053519:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575964.565611:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575964.565620:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575964.565622:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575964.565626:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575964.565630:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575964.565638:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12895
00000400:00000200:3.0:1788575964.565647:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575964.565654:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575964.565657:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575964.565661:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565664:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575964.565666:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565668:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575964.565673:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575964.565675:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575964.565678:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575964.565682:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575964.565685:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575964.565687:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575964.565690:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565692:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575964.565695:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565697:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575964.565700:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575964.565704:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565706:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575964.565709:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575964.565712:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575964.565715:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575964.565717:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575964.565719:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575964.565728:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000020:00000040:3.0:1788575964.565732:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575964.565735:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575964.565736:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575964.565739:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575964.565742:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575964.565745:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575964.565752:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788575964.565758:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575964.565767:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575964.565776:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575964.565780:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575964.565784:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575964.565809:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575964.565813:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575964.565816:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575964.565817:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575964.565819:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575964.565822:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575964.565824:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12895
00000400:00000200:3.0:1788575964.565828:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575964.565831:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575964.565849:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575964.565853:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565855:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575964.565857:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565858:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575964.565860:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575964.565862:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788575964.565863:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575964.565865:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575964.565867:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788575964.565869:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788575964.565871:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565873:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575964.565874:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565876:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575964.565878:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788575964.565880:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575964.565882:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575964.565884:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575964.565889:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575964.565891:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575964.565893:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575964.565897:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788575964.565900:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575964.565901:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575964.565905:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788575964.565908:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575964.566056:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575964.566069:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575964.566072:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575964.566078:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.566086:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575964.566088:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575964.566091:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575964.566093:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575964.566095:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575964.566098:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393248896:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575964.566102:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575964.566106:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575964.566108:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575964.566111:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575964.566116:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575964.566117:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575964.566122:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575964.566126:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575964.566132:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575964.566139:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393248896, portal 25
00000100:00000001:2.0:1788575964.566141:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575964.566142:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:2.0:1788575964.566148:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.566154:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575964.566157:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575964.566159:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575964.566162:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393248896, offset 0
00000400:00000200:2.0:1788575964.566168:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575964.566176:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575964.566183:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575964.566188:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575964.566192:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575964.566196:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575964.566198:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575964.566202:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331200:331200:256:4294967295] 192.168.202.110@tcp LPNI seq info [331157:331157:8:4294967295]
00000400:00000200:2.0:1788575964.566206:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575964.566215:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575964.566221:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575964.566226:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575964.566233:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575964.566237:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575964.566242:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575964.566245:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575964.566250:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575964.566252:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575964.566258:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575964.566262:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575964.566267:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575964.566270:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575964.566273:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575964.566277:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.566287:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575964.566290:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.566298:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575964.566300:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575964.566305:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575964.566308:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575964.566311:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307569081
00000100:00000010:2.0:1788575964.566317:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cc00.
00000100:00000001:2.0:1788575964.566323:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:3.0:1788575964.566380:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575964.566382:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575964.566388:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575964.566389:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575964.566415:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575964.566421:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788575964.566426:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575964.566431:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575964.566438:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575964.566440:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575964.566449:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575964.566457:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575964.566460:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575964.566463:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575964.566464:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575964.566466:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575964.566469:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575964.566471:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12895
00000400:00000200:3.0:1788575964.566475:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575964.566477:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575964.566480:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575964.566482:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.566484:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575964.566486:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.566488:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575964.566490:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575964.566491:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575964.566493:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575964.566495:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575964.566497:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575964.566499:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788575964.566502:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.566504:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575964.566505:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575964.566507:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575964.566509:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575964.566511:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575964.566514:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575964.566515:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575964.566519:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575964.566521:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575964.566523:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575964.566525:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575964.566528:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575964.566529:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575964.566534:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788575964.566536:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575964.566539:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575964.566540:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575964.566542:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575964.566543:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575964.566545:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575964.566549:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788575964.566552:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575964.566557:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575964.566564:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575964.566566:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575964.566570:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575964.566577:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575964.566580:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575964.566583:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575964.566587:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12905)
00000100:00000001:2.0:1788575964.566700:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307569082
00000100:00000010:2.0:1788575964.566705:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c3c0.
00000100:00000001:2.0:1788575964.566710:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575964.566713:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575964.566716:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575964.566718:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575964.566723:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575964/real 1788575964]  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575964.566734:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575964.566735:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575964.566738:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575964.566741:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575964.566742:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575964.566745:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575964.566749:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575964.566757:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.566763:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575964.566766:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.566768:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575964.566770:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575964.566774:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.566781:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575964.566786:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.566792:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575964.566795:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.566797:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575964.566798:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.566800:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575964.566802:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575964.566805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575964.566808:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575964.566811:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575964.566815:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575964.566822:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393248896:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575964.566826:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.566828:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575964.566831:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393248896/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.566847:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575964.566850:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575964.566852:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575964.566855:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575964.566857:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575964.566859:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575964.566861:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575964.566864:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788575964.566867:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575964.566869:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575964.566870:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:2.0:1788575964.566873:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575964.566874:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.566876:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.566883:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575964.566885:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575964.566887:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575964.566888:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575964.566889:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575964.566894:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.566899:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575964.566900:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575964.566902:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575964.566904:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575964.566906:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575964.566909:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249024:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575964.566994:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575964.566997:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575964.566998:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575964.567000:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575964.567004:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575964.567006:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575964.567010:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575964.567012:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575964.567016:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575964.567019:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393249024, portal 10
00000100:00000001:2.0:1788575964.567021:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575964.567022:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:2.0:1788575964.567029:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567036:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575964.567038:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575964.567040:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575964.567043:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393249024, offset 0
00000400:00000200:2.0:1788575964.567047:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575964.567051:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575964.567056:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575964.567059:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575964.567073:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575964.567075:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575964.567078:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575964.567081:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331201:331201:256:4294967295] 192.168.202.110@tcp LPNI seq info [331158:331158:8:4294967295]
00000400:00000200:2.0:1788575964.567085:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575964.567091:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575964.567095:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575964.567099:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575964.567103:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575964.567106:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575964.567115:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575964.567118:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575964.567122:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575964.567124:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575964.567127:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575964.567130:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575964.567133:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575964.567135:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575964.567137:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575964.567140:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567148:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575964.567151:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567163:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575964.567165:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575964.567168:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575964.567172:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575964.567175:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575964.567178:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307568167
00000100:00000001:2.0:1788575964.567180:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307569081
00000100:00000001:2.0:1788575964.567183:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307568167
00000100:00000001:2.0:1788575964.567184:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307569082
00000100:00000001:2.0:1788575964.567186:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575964.567188:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575964.567190:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575964.567194:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575964/real 1788575964]  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575964.567201:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575964.567202:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575964.567204:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575964.567206:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575964.567208:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575964.567209:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575964.567213:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575964.567222:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567229:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575964.567230:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567232:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575964.567234:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575964.567237:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567245:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575964.567248:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567255:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575964.567258:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567260:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575964.567261:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567262:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575964.567264:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575964.567267:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575964.567271:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575964.567273:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575964.567277:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575964.567285:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249024:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575964.567289:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575964.567292:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567298:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575964.567299:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575964.567302:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575964.567304:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575964.567306:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575964.567309:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249152:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575964.567312:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575964.567314:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575964.567316:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575964.567318:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575964.567321:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575964.567323:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575964.567326:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575964.567329:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575964.567332:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575964.567335:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393249152, portal 4
00000100:00000001:2.0:1788575964.567336:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575964.567338:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788575964.567343:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567349:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575964.567351:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575964.567354:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575964.567356:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393249152, offset 0
00000400:00000200:2.0:1788575964.567360:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575964.567363:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575964.567367:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575964.567369:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575964.567372:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575964.567374:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575964.567376:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575964.567379:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331202:331202:256:4294967295] 192.168.202.110@tcp LPNI seq info [331159:331159:8:4294967295]
00000400:00000200:2.0:1788575964.567383:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575964.567389:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575964.567411:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575964.567415:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575964.567419:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575964.567421:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1788575964.567425:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575964.567428:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1788575964.567431:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575964.567434:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575964.567436:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575964.567440:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575964.567443:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575964.567445:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575964.567448:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575964.567451:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567459:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575964.567461:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567481:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575964.567483:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575964.567487:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575964.567489:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575964.567493:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575964.567494:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307568313
00000100:00000001:2.0:1788575964.567497:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307569081
00000100:00000001:2.0:1788575964.567499:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307568313
00000100:00000001:2.0:1788575964.567501:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307569082
00000100:00000001:2.0:1788575964.567503:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575964.567505:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575964.567506:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575964.567510:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575964/real 1788575964]  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575964.567518:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575964.567519:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575964.567521:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575964.567523:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575964.567525:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575964.567528:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575964.567531:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575964.567540:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567547:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575964.567548:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567550:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575964.567551:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575964.567555:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567562:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575964.567566:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567572:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575964.567575:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567577:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575964.567578:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567579:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575964.567581:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575964.567584:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575964.567587:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575964.567589:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575964.567593:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575964.567601:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249152:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575964.567605:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567607:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575964.567611:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393249024/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567617:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575964.567618:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575964.567620:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788575964.567622:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575964.567624:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575964.567625:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575964.567627:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575964.567629:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788575964.567632:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575964.567633:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575964.567634:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:2.0:1788575964.567638:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575964.567639:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567641:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575964.567644:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393249152/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575964.567650:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575964.567651:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575964.567652:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788575964.567654:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575964.567656:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575964.567657:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575964.567659:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575964.567660:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788575964.567662:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575964.567663:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575964.567664:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788575964.567666:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575964.567667:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567669:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575964.567673:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575964.567674:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575964.567675:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575964.567677:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575964.949507:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575964.949513:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575964.949518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575964.949519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575964.949523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575964.949524:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575964.949585:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575964.949589:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575964.949592:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575965.077501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575965.077507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575965.077509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575965.077515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575965.077518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575965.077522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575965.077525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575965.077527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575965.077530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575965.525524:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307569082, at 4307569088
00000100:00000010:1.0:1788575965.525547:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c3c0.
00000100:00000040:0.0:1788575965.525734:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307569081, at 4307569088
00000100:00000010:0.0:1788575965.525740:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc00.
00010000:00000001:3.0:1788575965.973498:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575965.973504:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575965.973509:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575965.973510:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575965.973514:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575965.973515:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575965.973581:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575965.973585:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575965.973588:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575966.101441:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575966.101446:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575966.101448:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575966.101452:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575966.101454:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575966.101457:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575966.101459:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575966.101461:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575966.101463:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575966.997524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575966.997531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575966.997536:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575966.997549:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575966.997551:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575966.997554:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575966.997558:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575966.997560:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575966.997562:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575966.997564:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575966.997568:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575966.997571:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575966.997572:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575966.997573:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575966.997575:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575966.997576:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575966.997578:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575966.997579:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575966.998001:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575966.998004:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575966.998006:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575967.125496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575967.125503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575967.125506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575967.125511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575967.125515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575967.125518:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575967.125520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575967.125523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575967.125526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575968.021504:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575968.021509:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575968.021512:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575968.021513:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575968.021515:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575968.021516:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575968.021572:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575968.021576:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575968.021580:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575968.150623:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575968.150632:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575968.150634:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575968.150640:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575968.150643:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575968.150646:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575968.150648:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575968.150650:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575968.150653:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575969.045684:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575969.045692:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575969.045695:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575969.045698:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575969.045702:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575969.045704:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575969.045706:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575969.045709:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575969.045713:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575969.045714:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575969.045717:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575969.045719:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575969.045855:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575969.045859:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575969.045862:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575969.174357:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575969.174365:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575969.174367:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575969.174372:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575969.174376:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575969.174379:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.174382:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575969.174384:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.174387:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575969.685467:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575969.685476:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788575969.685479:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575969.685483:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575969.685488:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575969.685495:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12900
00000400:00000200:3.0:1788575969.685504:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575969.685510:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575969.685513:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575969.685517:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685519:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575969.685522:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685524:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575969.685528:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575969.685530:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788575969.685532:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575969.685536:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788575969.685538:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788575969.685540:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788575969.685544:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685546:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575969.685547:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685549:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575969.685553:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788575969.685556:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685558:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575969.685561:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575969.685564:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788575969.685567:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575969.685569:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788575969.685571:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575969.685580:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000020:00000040:3.0:1788575969.685583:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788575969.685587:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575969.685589:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575969.685591:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575969.685594:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575969.685598:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575969.685604:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788575969.685610:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788575969.685619:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788575969.685629:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575969.685634:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575969.685638:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788575969.685654:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575969.685657:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788575969.685659:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788575969.685660:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575969.685661:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788575969.685663:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788575969.685665:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12900
00000400:00000200:3.0:1788575969.685668:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575969.685670:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575969.685672:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788575969.685675:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685677:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575969.685678:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685680:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788575969.685681:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575969.685682:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575969.685683:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788575969.685683:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788575969.685685:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788575969.685688:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575969.685691:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788575969.685691:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:2.0:1788575969.685696:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788575969.685696:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685700:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788575969.685701:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685702:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575969.685706:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:2.0:1788575969.685707:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788575969.685709:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685713:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575969.685717:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575969.685722:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575969.685724:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1788575969.685725:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575969.685727:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575969.685728:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575969.685732:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788575969.685734:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:3.0:1788575969.685735:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575969.685738:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788575969.685738:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788575969.685742:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
02000000:00000001:2.0:1788575969.685743:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788575969.685745:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788575969.685748:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575969.685750:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575969.685752:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788575969.685752:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575969.685754:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575969.685756:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788575969.685759:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575969.685761:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393249280:192.168.202.110@tcp:250:kworker.0
00000100:00100000:3.0:1788575969.685764:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575969.685770:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1788575969.685772:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788575969.685774:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575969.685776:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:3.0:1788575969.685777:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575969.685780:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575969.685802:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575969.685806:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:2.0:1788575969.685807:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:3.0:1788575969.685809:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788575969.685812:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788575969.685814:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000010:2.0:1788575969.685815:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
00000100:00000001:3.0:1788575969.685818:0:188751:0:(import.c:520:import_select_connection()) Process entered
02000000:00000001:2.0:1788575969.685819:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788575969.685823:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12900
00000400:00000200:2.0:1788575969.685828:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788575969.685829:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575969.685836:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000200:3.0:1788575969.685836:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788575969.685842:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000010:2.0:1788575969.685846:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000040:3.0:1788575969.685847:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685852:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:2.0:1788575969.685854:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393249280, portal 25
00000100:00000040:3.0:1788575969.685855:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575969.685859:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788575969.685860:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575969.685863:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000020:00000001:3.0:1788575969.685864:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788575969.685867:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788575969.685871:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575969.685873:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788575969.685874:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788575969.685880:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788575969.685884:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788575969.685887:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788575969.685891:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575969.685895:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685896:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575969.685899:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000040:3.0:1788575969.685901:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575969.685905:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393249280, offset 0
00000100:00000001:3.0:1788575969.685906:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788575969.685912:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788575969.685917:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788575969.685919:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788575969.685920:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788575969.685922:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788575969.685924:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788575969.685924:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575969.685929:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788575969.685931:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788575969.685932:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575969.685933:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575969.685936:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1788575969.685937:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000020:00000040:3.0:1788575969.685941:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788575969.685943:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788575969.685944:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788575969.685945:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575969.685947:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:3.0:1788575969.685948:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788575969.685951:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788575969.685960:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788575969.685966:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575969.685969:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:3.0:1788575969.685974:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575969.685976:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788575969.685986:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575969.685988:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788575969.685990:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575969.685994:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:3.0:1788575969.685998:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575969.686003:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331203:331203:256:4294967295] 192.168.202.110@tcp LPNI seq info [331160:331160:8:4294967295]
00000400:00000200:2.0:1788575969.686015:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788575969.686015:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788575969.686020:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788575969.686025:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788575969.686032:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12910)
00000400:00000200:2.0:1788575969.686039:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575969.686046:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575969.686053:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575969.686060:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575969.686064:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575969.686069:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575969.686072:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575969.686076:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575969.686079:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575969.686084:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575969.686089:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575969.686094:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575969.686096:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575969.686107:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575969.686111:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686121:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575969.686128:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686135:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575969.686137:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575969.686143:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575969.686147:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575969.686156:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307574201
00000100:00000010:2.0:1788575969.686161:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c6c0.
00000100:00000001:2.0:1788575969.686174:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575969.686195:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307574201
00000100:00000010:2.0:1788575969.686199:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ca80.
00000100:00000001:2.0:1788575969.686202:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575969.686205:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575969.686207:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575969.686210:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575969.686215:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575969/real 1788575969]  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575969.686224:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575969.686226:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575969.686228:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575969.686230:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575969.686232:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575969.686234:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575969.686238:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575969.686245:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686263:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575969.686265:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686268:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575969.686270:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575969.686274:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686281:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575969.686291:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686298:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575969.686301:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686303:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575969.686304:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686306:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575969.686308:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575969.686329:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575969.686334:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575969.686336:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575969.686341:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575969.686349:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393249280:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575969.686359:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686362:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575969.686365:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393249280/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686371:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575969.686374:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575969.686376:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575969.686379:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575969.686381:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575969.686387:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575969.686390:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575969.686431:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788575969.686436:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575969.686437:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575969.686439:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788575969.686442:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575969.686443:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686445:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686451:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575969.686453:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575969.686454:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575969.686456:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575969.686457:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575969.686461:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686467:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575969.686468:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575969.686470:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575969.686472:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575969.686478:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575969.686481:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575969.686484:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575969.686486:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575969.686487:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575969.686489:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575969.686492:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575969.686494:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575969.686498:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575969.686501:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575969.686522:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575969.686525:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393249408, portal 10
00000100:00000001:2.0:1788575969.686528:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575969.686529:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788575969.686536:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686542:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575969.686544:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575969.686547:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575969.686549:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393249408, offset 0
00000400:00000200:2.0:1788575969.686553:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575969.686556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575969.686560:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575969.686563:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575969.686566:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575969.686568:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575969.686570:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575969.686573:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331204:331204:256:4294967295] 192.168.202.110@tcp LPNI seq info [331161:331161:8:4294967295]
00000400:00000200:2.0:1788575969.686577:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575969.686582:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575969.686594:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575969.686597:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575969.686601:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575969.686609:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575969.686613:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575969.686615:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575969.686618:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575969.686619:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575969.686621:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575969.686623:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575969.686626:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575969.686627:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575969.686628:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575969.686631:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686640:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575969.686642:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686645:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575969.686646:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575969.686648:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575969.686655:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575969.686658:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575969.686659:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307573287
00000100:00000001:2.0:1788575969.686661:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307574201
00000100:00000001:2.0:1788575969.686662:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307573287
00000100:00000001:2.0:1788575969.686664:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307574201
00000100:00000001:2.0:1788575969.686665:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575969.686666:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575969.686667:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575969.686669:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575969/real 1788575969]  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575969.686674:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575969.686674:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575969.686678:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575969.686680:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575969.686681:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575969.686682:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575969.686684:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575969.686689:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686692:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575969.686693:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686695:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575969.686696:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575969.686698:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686701:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575969.686703:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686710:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575969.686712:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686713:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575969.686713:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686714:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575969.686716:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575969.686717:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575969.686719:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575969.686721:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575969.686723:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575969.686730:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575969.686732:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575969.686734:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686738:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575969.686739:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575969.686740:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575969.686741:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575969.686742:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575969.686744:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575969.686746:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575969.686747:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575969.686748:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575969.686749:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575969.686751:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575969.686766:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575969.686769:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575969.686770:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575969.686773:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575969.686774:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393249536, portal 4
00000100:00000001:2.0:1788575969.686775:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575969.686776:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788575969.686780:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686784:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575969.686785:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575969.686797:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000200:2.0:1788575969.686800:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393249536, offset 0
00000400:00000200:2.0:1788575969.686803:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575969.686807:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575969.686810:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575969.686812:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575969.686813:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575969.686815:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575969.686816:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575969.686818:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331205:331205:256:4294967295] 192.168.202.110@tcp LPNI seq info [331162:331162:8:4294967295]
00000400:00000200:2.0:1788575969.686820:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575969.686825:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575969.686827:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575969.686829:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000800:00000200:2.0:1788575969.686832:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575969.686834:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1788575969.686836:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575969.686837:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1788575969.686840:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575969.686841:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:2.0:1788575969.686843:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575969.686859:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575969.686861:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:2.0:1788575969.686862:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:2.0:1788575969.686863:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575969.686866:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686870:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575969.686872:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686876:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575969.686877:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575969.686879:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575969.686880:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575969.686882:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575969.686883:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307573433
00000100:00000001:2.0:1788575969.686885:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307574201
00000100:00000001:2.0:1788575969.686887:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307573433
00000100:00000001:2.0:1788575969.686888:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307574201
00000100:00000001:2.0:1788575969.686889:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575969.686890:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575969.686891:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575969.686893:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575969/real 1788575969]  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575969.686897:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575969.686899:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575969.686900:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575969.686901:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575969.686902:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575969.686903:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575969.686905:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575969.686909:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686912:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575969.686913:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686914:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575969.686915:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575969.686917:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686920:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575969.686922:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686925:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575969.686927:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686928:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575969.686928:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686930:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575969.686931:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575969.686933:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575969.686934:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575969.686935:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575969.686937:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575969.686945:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575969.686947:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686948:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575969.686950:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393249408/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686953:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575969.686954:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575969.686955:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575969.686957:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575969.686957:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575969.686958:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575969.686959:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575969.686960:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788575969.686962:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575969.686962:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575969.686963:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788575969.686965:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575969.686966:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686967:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575969.686978:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393249536/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575969.686982:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575969.686983:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575969.686984:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788575969.686985:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575969.686986:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575969.686986:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575969.686987:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575969.686988:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788575969.686989:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575969.686990:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575969.686990:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788575969.686992:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575969.686993:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686994:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575969.686996:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575969.686997:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575969.686998:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575969.687000:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575970.070721:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575970.070728:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575970.070734:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575970.070735:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575970.070737:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575970.070738:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575970.070816:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575970.070820:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575970.070824:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575970.197652:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575970.197659:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575970.197661:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575970.197667:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575970.197670:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575970.197674:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575970.197676:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575970.197679:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575970.197682:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788575970.645505:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307574201, at 4307574208
00000100:00000010:1.0:1788575970.645521:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca80.
00000100:00000040:0.0:1788575970.645624:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307574201, at 4307574208
00000100:00000010:0.0:1788575970.645630:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c6c0.
00010000:00000001:3.0:1788575971.093504:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575971.093510:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575971.093516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575971.093517:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575971.093521:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575971.093522:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575971.093688:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575971.093692:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575971.093695:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575971.221511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575971.221524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575971.221526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575971.221531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575971.221534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575971.221538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575971.221540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575971.221542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575971.221545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575972.117523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575972.117530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575972.117535:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575972.117537:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575972.117540:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575972.117541:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575972.117597:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575972.117600:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575972.117604:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575972.245520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575972.245527:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575972.245530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575972.245535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575972.245539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575972.245544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575972.245546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575972.245549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575972.245552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575973.141528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575973.141534:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575973.141540:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575973.141541:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575973.141544:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575973.141545:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575973.141707:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575973.141710:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575973.141712:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575973.269534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575973.269542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575973.269544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575973.269550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575973.269554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575973.269558:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575973.269560:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575973.269563:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575973.269566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575974.165850:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575974.165856:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575974.165861:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575974.165862:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575974.165866:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575974.165867:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575974.165929:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575974.165932:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575974.165935:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575974.293530:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575974.293538:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575974.293540:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575974.293545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575974.293548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575974.293551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575974.293554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575974.293557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575974.293560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575974.805587:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575974.805597:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575974.805600:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575974.805603:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575974.805607:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575974.805614:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12905
00000400:00000200:0.0:1788575974.805622:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575974.805629:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575974.805631:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575974.805634:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575974.805637:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575974.805640:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575974.805642:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575974.805646:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575974.805648:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575974.805651:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575974.805654:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575974.805656:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575974.805659:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575974.805662:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575974.805664:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575974.805666:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575974.805668:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575974.805671:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575974.805675:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575974.805677:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575974.805680:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575974.805683:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575974.805686:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575974.805687:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575974.805689:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575974.805698:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788575974.805701:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575974.805705:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575974.805707:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575974.805709:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575974.805712:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575974.805716:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575974.805723:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788575974.805729:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575974.805738:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575974.805759:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575974.805764:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575974.805768:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575974.805789:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575974.805793:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575974.805796:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575974.805797:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575974.805799:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575974.805801:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575974.805804:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12905
00000400:00000200:0.0:1788575974.805808:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575974.805813:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1788575974.805819:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1788575974.805822:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575974.805823:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575974.805826:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575974.805832:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575974.805839:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575974.805842:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575974.805844:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575974.805845:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575974.805848:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575974.805850:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788575974.805854:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788575974.805854:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1788575974.805859:0:188751:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1788575974.805860:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575974.805862:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00100000:2.0:1788575974.805865:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393249664:192.168.202.110@tcp:250:kworker.0
00000020:00000001:0.0:1788575974.805866:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575974.805870:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575974.805871:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1788575974.805874:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575974.805876:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:0.0:1788575974.805878:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575974.805879:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575974.805883:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1788575974.805886:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000010:2.0:1788575974.805888:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
00000100:00000001:0.0:1788575974.805890:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575974.805892:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575974.805899:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575974.805901:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00000040:0.0:1788575974.805905:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:2.0:1788575974.805907:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000001:0.0:1788575974.805910:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575974.805914:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000200:2.0:1788575974.805914:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393249664, portal 25
00000100:00000001:2.0:1788575974.805918:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1788575974.805921:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575974.805922:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00000001:0.0:1788575974.805928:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575974.805931:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575974.805932:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575974.805938:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1788575974.805940:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575974.805942:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788575974.805945:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575974.805946:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65990.
00000100:00000001:0.0:1788575974.805950:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:2.0:1788575974.805951:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393249664, offset 0
00000008:00100000:0.0:1788575974.805955:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1788575974.805959:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00000001:0.0:1788575974.805962:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575974.805965:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575974.805968:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1788575974.805970:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575974.805974:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000400:00000200:2.0:1788575974.805975:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1788575974.805979:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575974.805983:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575974.805983:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575974.805987:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1788575974.805988:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:0.0:1788575974.805990:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575974.805993:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1788575974.805994:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575974.805997:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575974.806001:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331206:331206:256:4294967295] 192.168.202.110@tcp LPNI seq info [331163:331163:8:4294967295]
02000000:00000010:0.0:1788575974.806002:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000400:00000200:2.0:1788575974.806007:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575974.806008:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575974.806018:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575974.806018:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575974.806023:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575974.806027:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133300.
00000100:00000001:0.0:1788575974.806037:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575974.806041:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788575974.806042:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1788575974.806048:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1788575974.806054:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575974.806063:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575974.806066:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575974.806070:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575974.806071:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575974.806073:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575974.806076:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000001:2.0:1788575974.806076:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575974.806081:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12905
00000400:00000200:2.0:1788575974.806084:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575974.806089:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575974.806090:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307579321
00000400:00000200:0.0:1788575974.806094:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575974.806098:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000010:2.0:1788575974.806099:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c2a0.
00000100:00000040:0.0:1788575974.806103:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575974.806107:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575974.806108:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:0.0:1788575974.806110:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575974.806115:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575974.806115:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307579321
00000020:00000001:0.0:1788575974.806120:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000010:2.0:1788575974.806121:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c300.
00000020:00000040:0.0:1788575974.806124:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:2.0:1788575974.806125:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000001:0.0:1788575974.806129:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788575974.806130:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1788575974.806132:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:2.0:1788575974.806133:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575974.806138:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788575974.806139:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575974.806144:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788575974.806145:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575974.806151:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575974.806153:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575974.806154:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575974.806156:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000040:0.0:1788575974.806158:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575974.806159:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575974.806162:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575974.806163:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575974.806164:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1788575974.806169:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000040:2.0:1788575974.806170:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1788575974.806175:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575974.806180:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575974.806183:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575974.806187:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575974.806191:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575974.806195:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575974.806198:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575974.806200:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:0.0:1788575974.806202:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575974.806204:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575974.806206:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575974.806208:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:0.0:1788575974.806211:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
02000000:00000001:2.0:1788575974.806212:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1788575974.806216:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788575974.806218:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575974.806221:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575974.806224:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575974.806225:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249792:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575974.806227:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1788575974.806230:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575974.806231:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575974.806233:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575974.806235:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575974.806236:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575974.806239:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575974.806243:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788575974.806247:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575974.806251:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
00000100:00100000:0.0:1788575974.806253:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1788575974.806256:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575974.806266:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575974.806267:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575974.806271:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788575974.806274:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000100:00000040:0.0:1788575974.806278:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788575974.806281:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575974.806287:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393249792, portal 10
00000100:00000001:2.0:1788575974.806293:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575974.806294:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575974.806295:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00000040:0.0:1788575974.806301:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788575974.806306:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788575974.806307:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575974.806315:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12916)
00000100:00000001:2.0:1788575974.806316:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575974.806320:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575974.806323:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65aa0.
00000100:00000200:2.0:1788575974.806331:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393249792, offset 0
00000400:00000200:2.0:1788575974.806335:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575974.806338:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575974.806341:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575974.806343:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575974.806345:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575974.806347:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575974.806348:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575974.806351:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331207:331207:255:4294967295] 192.168.202.110@tcp LPNI seq info [331164:331164:7:4294967295]
00000400:00000200:2.0:1788575974.806354:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575974.806360:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575974.806365:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575974.806368:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133b00.
00000800:00000200:2.0:1788575974.806375:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575974.806380:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575974.806383:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575974.806386:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575974.806390:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575974.806410:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307578407
00000100:00000001:2.0:1788575974.806412:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307579321
00000100:00000001:2.0:1788575974.806413:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307578407
00000100:00000001:2.0:1788575974.806415:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307579321
00000100:00000001:2.0:1788575974.806416:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575974.806417:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575974.806419:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575974.806420:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575974.806423:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575974.806423:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575974.806425:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575974.806426:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575974.806426:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575974.806430:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575974.806436:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575974.806444:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575974.806446:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575974.806448:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575974.806450:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575974.806453:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249920:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575974.806459:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575974.806461:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575974.806463:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575974.806465:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575974.806469:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575974.806472:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575974.806476:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575974.806480:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788575974.806484:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65660.
00000100:00000200:2.0:1788575974.806487:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393249920, portal 4
00000100:00000001:2.0:1788575974.806489:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575974.806490:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:2.0:1788575974.806498:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575974.806504:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575974.806506:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575974.806510:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65e58.
00000100:00000200:2.0:1788575974.806513:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393249920, offset 0
00000400:00000200:2.0:1788575974.806517:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575974.806522:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575974.806525:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575974.806528:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575974.806531:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575974.806533:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575974.806535:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575974.806538:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331208:331208:254:4294967295] 192.168.202.110@tcp LPNI seq info [331165:331165:6:4294967295]
00000400:00000200:2.0:1788575974.806542:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575974.806548:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575974.806552:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575974.806555:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133600.
00000800:00000200:2.0:1788575974.806558:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575974.806569:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788575974.806571:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575974.806573:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575974.806579:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575974.806581:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307578553
00000100:00000001:2.0:1788575974.806583:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307579321
00000100:00000001:2.0:1788575974.806588:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307578553
00000100:00000001:2.0:1788575974.806590:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307579321
00000100:00000001:2.0:1788575974.806593:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575974.806596:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575974.806599:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575974.806601:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575974.806604:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575974.806605:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788575974.806607:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575974.806608:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575974.806610:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575974.806611:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575975.189529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575975.189537:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575975.189542:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575975.189543:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575975.189547:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575975.189548:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575975.189598:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575975.189601:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575975.189604:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575975.317516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575975.317523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575975.317525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575975.317531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575975.317534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575975.317538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575975.317541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575975.317543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575975.317546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575975.765475:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307579321, at 4307579328
00000100:00000040:1.0:1788575975.765475:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307579321, at 4307579328
00000100:00000010:0.0:1788575975.765488:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c2a0.
00000100:00000010:1.0:1788575975.765490:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c300.
00010000:00000001:3.0:1788575976.213495:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575976.213500:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575976.213503:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575976.213504:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575976.213506:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575976.213507:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575976.213560:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575976.213565:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575976.213568:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575976.341500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575976.341505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575976.341507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575976.341511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575976.341513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575976.341516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575976.341518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575976.341520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575976.341521:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575977.237571:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575977.237578:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575977.237589:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575977.237592:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575977.237595:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575977.237598:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575977.237613:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575977.237615:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575977.237617:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575977.237619:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575977.237623:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575977.237625:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575977.237626:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575977.237627:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575977.237629:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575977.237630:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575977.237632:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575977.237634:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575977.237703:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575977.237707:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575977.237711:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575977.365507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575977.365513:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575977.365515:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575977.365520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575977.365523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575977.365527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575977.365529:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575977.365531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575977.365534:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788575977.877797:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788575977.877838:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788575977.877861:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575977.877865:0:2351:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788575977.877871:0:2351:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788575977.877874:0:2351:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788575977.877879:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575977.877883:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133300.
00000400:00000200:1.0:1788575977.877888:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575977.877892:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575977.877899:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65990
00000400:00000010:1.0:1788575977.877901:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65990.
00000100:00000001:1.0:1788575977.877907:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575977.877916:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575977.877945:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575977.877949:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575977.877956:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575977.877958:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575977.877968:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575977.877970:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133b00.
00000400:00000200:1.0:1788575977.877972:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575977.877976:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575977.877978:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65aa0
00000400:00000010:1.0:1788575977.877980:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65aa0.
00000100:00000001:1.0:1788575977.877982:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575977.877985:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575977.877992:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575977.877995:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575977.878001:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575977.878003:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788575977.878007:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788575977.878009:0:2351:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133600.
00000400:00000200:1.0:1788575977.878011:0:2351:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788575977.878014:0:2351:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788575977.878017:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65e58
00000400:00000010:1.0:1788575977.878018:0:2351:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65e58.
00000100:00000001:1.0:1788575977.878020:0:2351:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788575977.878024:0:2351:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575977.878031:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788575977.878034:0:2351:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788575977.878041:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788575977.878042:0:2351:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575977.878164:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575977.878170:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575977.878174:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575977.878181:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575974/real 1788575977]  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575977.878193:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575977.878195:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575977.878198:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fdc0.
00000400:00000200:2.0:1788575977.878202:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c65660
00000400:00000010:2.0:1788575977.878203:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c65660.
00000100:00000001:2.0:1788575977.878206:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575977.878210:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575977.878217:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878229:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575977.878231:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878234:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575977.878236:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575977.878240:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878246:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575977.878251:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878257:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575977.878260:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878262:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575977.878263:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878265:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575977.878267:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575977.878271:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575977.878275:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575977.878278:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575977.878282:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575977.878289:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249920:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575977.878293:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575977.878296:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575974/real 1788575977]  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575977.878302:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575977.878303:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575977.878305:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119f6e0.
00000400:00000200:2.0:1788575977.878307:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575977.878308:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575977.878310:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575977.878312:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575977.878319:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878324:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575977.878325:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878326:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575977.878327:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575977.878331:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878336:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575977.878339:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878345:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575977.878347:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878348:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575977.878349:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878351:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575977.878353:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575977.878355:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575977.878358:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575977.878360:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575977.878363:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575977.878370:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393249792:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575977.878373:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575977.878377:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575974/real 1788575977]  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575977.878384:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575977.878385:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575977.878386:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575977.878389:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575977.878879:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575977.878884:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575977.878889:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575977.878897:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878903:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575977.878904:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878907:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575977.878908:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575977.878911:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878917:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575977.878921:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878926:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575977.878928:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878929:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575977.878930:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878932:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575977.878934:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575977.878937:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575977.878941:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575977.878943:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575977.878946:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575977.878954:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393249664:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575977.878958:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.878960:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575977.878963:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393249920/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.878970:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575977.878973:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575977.878977:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575977.878980:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575977.878983:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575977.878985:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575977.878986:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575977.878989:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:2.0:1788575977.878992:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575977.878994:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575977.878996:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:2.0:1788575977.878999:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575977.879000:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.879002:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575977.879005:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393249792/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.879011:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575977.879012:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575977.879013:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575977.879021:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575977.879022:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575977.879023:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575977.879024:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575977.879026:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:2.0:1788575977.879028:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575977.879029:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575977.879031:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:2.0:1788575977.879033:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575977.879034:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.879035:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575977.879037:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393249664/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575977.879041:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575977.879042:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575977.879044:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575977.879045:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575977.879046:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575977.879047:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575977.879048:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575977.879050:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:2.0:1788575977.879051:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575977.879052:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575977.879053:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:2.0:1788575977.879055:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575977.879056:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.879057:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575977.879064:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575977.879066:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575977.879068:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575977.879069:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575978.261523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575978.261529:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575978.261534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575978.261535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575978.261539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575978.261540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575978.261605:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575978.261609:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575978.261612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788575978.389516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788575978.389523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788575978.389525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788575978.389530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788575978.389534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788575978.389538:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575978.389541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788575978.389544:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788575978.389546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575979.285516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575979.285523:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575979.285525:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575979.285528:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575979.285533:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575979.285535:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575979.285537:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575979.285539:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575979.285542:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575979.285544:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575979.285547:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575979.285548:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575979.285643:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575979.285647:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575979.285650:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575979.413529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575979.413536:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575979.413538:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575979.413543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575979.413546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575979.413549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.413551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575979.413553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.413554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575979.925517:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575979.925527:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575979.925530:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575979.925533:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575979.925538:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575979.925545:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12911
00000400:00000200:0.0:1788575979.925554:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575979.925559:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575979.925562:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575979.925565:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925568:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575979.925571:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925573:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575979.925577:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575979.925579:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575979.925583:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575979.925586:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575979.925589:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000800:00000200:3.0:1788575979.925589:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000020:00000001:0.0:1788575979.925595:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000800:00000200:3.0:1788575979.925600:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000100:00000040:0.0:1788575979.925603:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925607:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1788575979.925607:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000100:00000040:0.0:1788575979.925619:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925626:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575979.925630:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575979.925645:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925649:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575979.925652:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575979.925663:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575979.925667:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575979.925669:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575979.925671:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575979.925678:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788575979.925682:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575979.925685:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575979.925687:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575979.925690:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575979.925697:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575979.925701:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575979.925709:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788575979.925716:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575979.925725:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575979.925736:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575979.925741:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575979.925745:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575979.925773:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575979.925777:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575979.925781:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575979.925783:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575979.925784:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575979.925787:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575979.925790:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12911
00000400:00000200:0.0:1788575979.925795:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575979.925798:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575979.925800:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575979.925803:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925805:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575979.925807:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925810:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:1.0:1788575979.925810:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:0.0:1788575979.925814:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575979.925817:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000800:00000001:1.0:1788575979.925818:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000020:00000001:0.0:1788575979.925819:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575979.925823:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000800:00000001:1.0:1788575979.925833:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1788575979.925835:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788575979.925839:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788575979.925842:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925844:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575979.925846:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000001:1.0:1788575979.925848:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000100:00000001:0.0:1788575979.925849:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575979.925852:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000800:00000001:1.0:1788575979.925853:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1788575979.925857:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925866:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575979.925868:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575979.925874:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575979.925876:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575979.925879:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575979.925883:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788575979.925885:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575979.925887:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575979.925891:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788575979.925893:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575979.925895:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575979.925897:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575979.925899:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575979.925900:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575979.925930:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575979.925935:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788575979.925938:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575979.925943:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575979.925950:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575979.925957:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575979.925960:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575979.925968:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575979.925970:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575979.925973:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788575979.925974:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575979.925976:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575979.925978:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575979.925980:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12911
00000400:00000200:0.0:1788575979.925984:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575979.925987:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575979.925989:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575979.925992:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925994:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575979.925996:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.925998:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575979.925999:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575979.926000:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788575979.926002:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575979.926004:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575979.926006:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575979.926008:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575979.926011:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.926013:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575979.926015:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575979.926017:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575979.926019:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575979.926022:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575979.926024:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575979.926026:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575979.926028:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575979.926030:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575979.926032:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575979.926035:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575979.926037:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575979.926038:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575979.926042:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788575979.926044:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575979.926047:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575979.926048:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575979.926049:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575979.926050:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575979.926052:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575979.926055:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788575979.926058:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575979.926062:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575979.926068:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575979.926069:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575979.926073:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575979.926079:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575979.926083:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788575979.926086:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788575979.926091:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12921)
00000100:00000001:2.0:1788575979.926573:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575979.926578:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575979.926582:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575979.926590:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.926598:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575979.926599:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575979.926603:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575979.926606:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575979.926608:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575979.926611:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393250048:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575979.926615:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575979.926620:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575979.926622:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575979.926624:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575979.926636:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575979.926638:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575979.926643:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575979.926647:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575979.926654:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575979.926669:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393250048, portal 25
00000100:00000001:2.0:1788575979.926671:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575979.926673:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523500544 : -130484186051072 : ffff895348710e00)
00000100:00000040:2.0:1788575979.926679:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.926685:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575979.926687:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575979.926690:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575979.926693:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393250048, offset 0
00000400:00000200:2.0:1788575979.926699:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575979.926706:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575979.926712:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575979.926718:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575979.926722:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575979.926725:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575979.926728:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575979.926731:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331209:331209:256:4294967295] 192.168.202.110@tcp LPNI seq info [331166:331166:8:4294967295]
00000400:00000200:2.0:1788575979.926735:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575979.926743:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575979.926748:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575979.926752:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788575979.926758:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575979.926761:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575979.926764:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575979.926796:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575979.926800:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575979.926802:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788575979.926806:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575979.926811:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575979.926816:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575979.926819:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575979.926823:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575979.926827:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.926835:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575979.926838:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.926844:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575979.926846:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575979.926853:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575979.926857:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575979.926860:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307584442
00000100:00000010:2.0:1788575979.926865:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c1e0.
00000100:00000001:2.0:1788575979.926871:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575979.926878:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307584442
00000100:00000010:2.0:1788575979.926880:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c7e0.
00000100:00000001:2.0:1788575979.926884:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575979.926887:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575979.926890:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575979.926893:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575979.926898:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575979/real 1788575979]  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575979.926918:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575979.926920:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575979.926922:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575979.926925:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575979.926927:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575979.926929:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575979.926933:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575979.926941:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.926947:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575979.926949:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.926951:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575979.926953:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575979.926956:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.926962:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575979.926966:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.926972:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575979.926975:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.926976:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575979.926977:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.926979:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575979.926981:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575979.926984:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575979.926988:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575979.926990:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575979.926994:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575979.927002:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393250048:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575979.927007:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575979.927010:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927016:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575979.927017:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575979.927019:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575979.927021:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575979.927022:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575979.927025:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250176:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575979.927027:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575979.927029:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575979.927031:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575979.927033:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575979.927036:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575979.927038:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575979.927041:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575979.927043:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575979.927047:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575979.927050:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393250176, portal 10
00000100:00000001:2.0:1788575979.927052:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575979.927053:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523505920 : -130484186045696 : ffff895348712300)
00000100:00000040:2.0:1788575979.927059:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927066:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575979.927069:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575979.927071:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575979.927073:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393250176, offset 0
00000400:00000200:2.0:1788575979.927077:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575979.927081:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575979.927084:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575979.927087:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575979.927089:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575979.927091:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575979.927093:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575979.927096:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331210:331210:256:4294967295] 192.168.202.110@tcp LPNI seq info [331167:331167:8:4294967295]
00000400:00000200:2.0:1788575979.927100:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575979.927106:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575979.927110:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575979.927113:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788575979.927116:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575979.927119:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575979.927123:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575979.927125:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575979.927128:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575979.927130:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788575979.927133:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575979.927136:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575979.927139:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575979.927140:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575979.927143:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575979.927146:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927154:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575979.927158:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927164:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575979.927167:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575979.927170:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575979.927190:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575979.927195:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575979.927197:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307583527
00000100:00000001:2.0:1788575979.927200:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307584442
00000100:00000001:2.0:1788575979.927202:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307583527
00000100:00000001:2.0:1788575979.927204:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307584442
00000100:00000001:2.0:1788575979.927206:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575979.927208:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575979.927210:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575979.927213:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575979/real 1788575979]  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575979.927222:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575979.927224:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575979.927225:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575979.927228:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575979.927229:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575979.927231:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575979.927234:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575979.927241:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927247:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575979.927248:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927249:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575979.927251:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575979.927254:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927259:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575979.927264:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927270:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575979.927273:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927274:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575979.927276:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927277:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575979.927279:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575979.927282:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575979.927285:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575979.927287:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575979.927291:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575979.927299:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250176:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575979.927303:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575979.927306:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927311:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575979.927313:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575979.927315:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575979.927317:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575979.927319:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575979.927322:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575979.927325:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575979.927327:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575979.927328:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575979.927330:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575979.927334:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575979.927335:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575979.927340:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575979.927342:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575979.927347:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575979.927350:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393250304, portal 4
00000100:00000001:2.0:1788575979.927351:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575979.927353:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00000040:2.0:1788575979.927358:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927364:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575979.927366:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575979.927368:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575979.927370:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393250304, offset 0
00000400:00000200:2.0:1788575979.927374:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575979.927378:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575979.927382:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575979.927384:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575979.927387:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575979.927389:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575979.927439:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575979.927443:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331211:331211:256:4294967295] 192.168.202.110@tcp LPNI seq info [331168:331168:8:4294967295]
00000400:00000200:2.0:1788575979.927448:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575979.927453:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575979.927458:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575979.927461:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788575979.927465:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575979.927468:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1788575979.927472:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575979.927474:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1788575979.927478:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575979.927480:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788575979.927482:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575979.927486:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575979.927489:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575979.927491:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575979.927494:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575979.927497:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927504:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575979.927507:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927514:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575979.927516:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575979.927519:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575979.927521:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575979.927524:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575979.927526:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307583674
00000100:00000001:2.0:1788575979.927529:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307584442
00000100:00000001:2.0:1788575979.927531:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307583674
00000100:00000001:2.0:1788575979.927532:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307584442
00000100:00000001:2.0:1788575979.927534:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575979.927535:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575979.927536:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575979.927539:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575979/real 1788575979]  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575979.927544:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575979.927545:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575979.927546:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575979.927548:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575979.927549:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575979.927550:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575979.927552:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575979.927557:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927561:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575979.927562:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927563:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575979.927564:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575979.927566:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927569:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575979.927571:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927575:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575979.927576:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927577:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575979.927578:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927579:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575979.927580:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575979.927582:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575979.927584:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575979.927585:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575979.927587:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575979.927592:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575979.927595:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927596:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575979.927598:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710e00 x1875444393250048/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927601:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575979.927603:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575979.927606:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575979.927608:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575979.927610:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575979.927611:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575979.927613:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575979.927615:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:2.0:1788575979.927619:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575979.927622:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575979.927624:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710e00.
00000100:00000001:2.0:1788575979.927633:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575979.927634:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927636:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575979.927639:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712300 x1875444393250176/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927645:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575979.927647:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575979.927648:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575979.927650:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575979.927652:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575979.927653:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575979.927655:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575979.927656:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:2.0:1788575979.927659:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575979.927660:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575979.927661:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712300.
00000100:00000001:2.0:1788575979.927663:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575979.927665:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927666:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575979.927669:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393250304/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575979.927674:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575979.927676:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575979.927677:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788575979.927680:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575979.927681:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575979.927682:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575979.927684:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575979.927686:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788575979.927688:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575979.927689:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575979.927690:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:2.0:1788575979.927692:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575979.927693:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927695:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575979.927701:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575979.927702:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575979.927704:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575979.927705:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575980.309515:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575980.309521:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575980.309527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575980.309528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575980.309532:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575980.309533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575980.309588:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575980.309592:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575980.309596:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575980.437645:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575980.437651:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575980.437654:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575980.437659:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575980.437662:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575980.437664:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575980.437666:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575980.437668:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575980.437670:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575980.885509:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307584442, at 4307584448
00000100:00000040:1.0:1788575980.885509:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307584442, at 4307584448
00000100:00000010:0.0:1788575980.885525:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c1e0.
00000100:00000010:1.0:1788575980.885527:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c7e0.
00010000:00000001:3.0:1788575981.333543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575981.333553:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575981.333560:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575981.333561:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575981.333565:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575981.333566:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575981.333643:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575981.333647:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575981.333651:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575981.461468:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575981.461476:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575981.461479:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575981.461491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575981.461495:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575981.461499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575981.461503:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575981.461506:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575981.461510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575982.357538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575982.357543:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575982.357547:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575982.357547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575982.357549:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575982.357550:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575982.357662:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575982.357665:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575982.357668:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575982.485509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575982.486678:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575982.486681:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575982.486686:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575982.486689:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575982.486693:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575982.486695:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575982.486697:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575982.486711:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575983.381519:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575983.381525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575983.381530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575983.381532:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575983.381535:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575983.381536:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575983.381629:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575983.381633:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575983.381635:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575983.509526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575983.509533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575983.509535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575983.509540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575983.509543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575983.509547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575983.509550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575983.509553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575983.509556:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575984.405475:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575984.405479:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575984.405482:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575984.405483:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575984.405491:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575984.405492:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575984.405545:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575984.405549:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575984.405553:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575984.533535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575984.533542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575984.533544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575984.533550:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575984.533554:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575984.533559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575984.533561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575984.533564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575984.533566:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575985.045499:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575985.045508:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575985.045510:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575985.045514:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575985.045518:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575985.045524:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12916
00000400:00000200:0.0:1788575985.045532:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575985.045538:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575985.045541:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575985.045543:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575985.045545:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575985.045547:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575985.045548:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575985.045551:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575985.045553:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575985.045555:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575985.045557:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575985.045558:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575985.045560:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575985.045562:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575985.045563:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575985.045564:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575985.045566:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575985.045568:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575985.045570:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575985.045571:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575985.045573:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575985.045576:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575985.045579:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575985.045581:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575985.045583:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575985.045592:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788575985.045595:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575985.045598:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575985.045600:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575985.045602:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575985.045605:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575985.045608:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575985.045615:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788575985.045622:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575985.045632:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575985.045642:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575985.045646:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575985.045650:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575985.045676:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575985.045680:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575985.045683:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575985.045685:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575985.045687:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575985.045691:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575985.045694:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12916
00000400:00000200:0.0:1788575985.045699:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575985.045702:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575985.045705:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575985.045707:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1788575985.045709:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575985.045711:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575985.045714:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788575985.045715:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575985.045721:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575985.045722:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575985.045723:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575985.045730:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575985.045734:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575985.045735:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575985.045738:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575985.045740:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1788575985.045742:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
02000000:00000001:2.0:1788575985.045743:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575985.045747:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:2.0:1788575985.045748:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393250432:192.168.202.110@tcp:250:kworker.0
00000020:00000040:0.0:1788575985.045752:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788575985.045752:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575985.045756:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:0.0:1788575985.045757:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575985.045759:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575985.045763:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575985.045763:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575985.045768:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
00000100:00000040:0.0:1788575985.045770:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575985.045771:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575985.045777:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575985.045780:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575985.045784:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00000040:0.0:1788575985.045790:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575985.045794:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:2.0:1788575985.045794:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000020:00000040:0.0:1788575985.045800:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000200:2.0:1788575985.045800:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393250432, portal 25
00000100:00000001:2.0:1788575985.045804:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575985.045806:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00080000:0.0:1788575985.045807:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575985.045812:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575985.045813:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575985.045817:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575985.045824:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575985.045825:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1788575985.045829:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575985.045831:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575985.045834:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:2.0:1788575985.045836:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000008:00100000:0.0:1788575985.045840:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000200:2.0:1788575985.045842:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393250432, offset 0
00000008:00000001:0.0:1788575985.045845:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575985.045849:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575985.045853:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000010:0.0:1788575985.045856:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788575985.045859:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575985.045862:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1788575985.045864:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575985.045865:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575985.045868:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575985.045870:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1788575985.045873:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1788575985.045874:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575985.045881:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000400:00000200:2.0:1788575985.045882:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1788575985.045888:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575985.045888:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575985.045895:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00100000:0.0:1788575985.045898:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575985.045901:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575985.045909:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331212:331212:256:4294967295] 192.168.202.110@tcp LPNI seq info [331169:331169:8:4294967295]
00000100:00000001:0.0:1788575985.045917:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575985.045918:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575985.045923:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575985.045929:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575985.045934:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575985.045945:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788575985.045951:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1788575985.045952:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000100:00080000:0.0:1788575985.045955:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575985.045961:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:2.0:1788575985.045962:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788575985.045965:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575985.045968:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:2.0:1788575985.045969:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:0.0:1788575985.045973:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:2.1:1788575985.045977:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1788575985.045980:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12916
00000800:00000200:2.1:1788575985.045982:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000400:00000200:0.0:1788575985.045988:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000100:2.0:1788575985.045992:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575985.045997:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1788575985.045997:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000100:00080000:0.0:1788575985.046003:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575985.046005:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1788575985.046007:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575985.046012:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788575985.046013:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1788575985.046017:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575985.046021:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788575985.046021:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575985.046026:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000020:00000001:0.0:1788575985.046028:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575985.046031:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:2.0:1788575985.046032:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000020:00000001:0.0:1788575985.046034:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575985.046038:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000200:2.0:1788575985.046040:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1788575985.046042:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575985.046048:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575985.046056:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575985.046071:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575985.046074:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575985.046075:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575985.046078:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575985.046081:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575985.046083:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575985.046085:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575985.046085:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1788575985.046091:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788575985.046092:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575985.046093:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575985.046098:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575985.046102:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000001:2.0:1788575985.046107:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788575985.046108:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046111:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575985.046112:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575985.046117:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000100:00000001:2.0:1788575985.046119:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575985.046120:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:2.0:1788575985.046125:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788575985.046126:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1788575985.046129:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307589561
02000000:00000001:0.0:1788575985.046130:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575985.046134:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:2.0:1788575985.046135:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cba0.
02000000:00000001:0.0:1788575985.046136:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575985.046139:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046145:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000010:0.0:1788575985.046147:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788575985.046152:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046153:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307589561
00000100:00000010:2.0:1788575985.046159:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cc60.
00000100:00100000:0.0:1788575985.046160:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575985.046165:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788575985.046170:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575985.046173:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575985.046174:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575985.046178:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046179:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1788575985.046187:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575985.046188:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575985/real 1788575985]  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575985.046213:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1788575985.046216:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1788575985.046217:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:2.0:1788575985.046219:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1788575985.046223:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:2.0:1788575985.046224:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575985.046228:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000100:00000040:0.0:1788575985.046232:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12926)
00000400:00000010:2.0:1788575985.046232:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575985.046238:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575985.046242:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575985.046248:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046253:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575985.046255:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046256:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575985.046258:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575985.046261:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046265:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575985.046268:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046272:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575985.046274:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046275:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575985.046276:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046277:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575985.046278:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575985.046280:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575985.046282:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575985.046283:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575985.046285:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575985.046290:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393250432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575985.046293:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046294:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575985.046296:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393250432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046300:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575985.046301:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575985.046304:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575985.046306:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575985.046307:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575985.046308:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575985.046309:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575985.046311:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:2.0:1788575985.046314:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575985.046315:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575985.046317:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:2.0:1788575985.046319:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575985.046319:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046321:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046326:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575985.046327:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046328:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575985.046329:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575985.046331:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575985.046333:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046337:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575985.046338:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575985.046339:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575985.046340:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575985.046341:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575985.046343:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575985.046345:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575985.046346:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575985.046347:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575985.046348:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575985.046350:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575985.046352:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575985.046354:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575985.046356:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575985.046358:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575985.046360:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393250560, portal 10
00000100:00000001:2.0:1788575985.046362:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575985.046363:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:2.0:1788575985.046367:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046372:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575985.046374:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575985.046376:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575985.046379:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393250560, offset 0
00000400:00000200:2.0:1788575985.046382:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575985.046386:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575985.046389:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575985.046413:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575985.046416:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575985.046419:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575985.046421:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575985.046423:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331213:331213:256:4294967295] 192.168.202.110@tcp LPNI seq info [331170:331170:8:4294967295]
00000400:00000200:2.0:1788575985.046426:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575985.046431:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575985.046434:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575985.046437:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788575985.046439:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575985.046441:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575985.046444:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575985.046445:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575985.046448:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575985.046449:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788575985.046451:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575985.046453:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575985.046455:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575985.046456:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575985.046458:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575985.046460:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046474:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575985.046476:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046479:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046480:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575985.046482:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575985.046484:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575985.046486:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575985.046488:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307588647
00000100:00000001:2.0:1788575985.046490:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307589561
00000100:00000001:2.0:1788575985.046491:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307588647
00000100:00000001:2.0:1788575985.046493:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307589561
00000100:00000001:2.0:1788575985.046494:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575985.046495:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046496:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575985.046498:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575985/real 1788575985]  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575985.046503:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575985.046504:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575985.046505:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575985.046506:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575985.046507:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575985.046508:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575985.046510:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575985.046515:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046518:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575985.046519:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046520:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575985.046521:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575985.046523:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046527:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575985.046529:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046533:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575985.046534:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046535:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575985.046535:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046536:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575985.046538:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575985.046539:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575985.046541:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575985.046542:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575985.046544:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575985.046548:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575985.046550:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575985.046552:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046558:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575985.046559:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575985.046561:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575985.046563:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575985.046565:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575985.046568:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250688:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575985.046571:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575985.046572:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575985.046574:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575985.046576:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575985.046579:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575985.046581:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575985.046584:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575985.046587:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575985.046590:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575985.046592:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393250688, portal 4
00000100:00000001:2.0:1788575985.046594:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575985.046596:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:2.0:1788575985.046601:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046605:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575985.046606:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575985.046608:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575985.046609:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393250688, offset 0
00000400:00000200:2.0:1788575985.046611:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575985.046614:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575985.046616:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575985.046618:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575985.046619:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575985.046621:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575985.046622:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575985.046624:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331214:331214:256:4294967295] 192.168.202.110@tcp LPNI seq info [331171:331171:8:4294967295]
00000400:00000200:2.0:1788575985.046626:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575985.046630:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575985.046633:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575985.046635:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788575985.046640:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575985.046642:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1788575985.046646:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575985.046648:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1788575985.046651:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575985.046653:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788575985.046655:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575985.046658:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575985.046661:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575985.046663:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575985.046665:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575985.046668:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576010 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046676:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575985.046679:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576010 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046683:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046684:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575985.046687:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575985.046688:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575985.046690:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575985.046692:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307588793
00000100:00000001:2.0:1788575985.046693:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307589561
00000100:00000001:2.0:1788575985.046695:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307588793
00000100:00000001:2.0:1788575985.046696:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307589561
00000100:00000001:2.0:1788575985.046697:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575985.046698:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046700:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575985.046702:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575985/real 1788575985]  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575985.046706:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575985.046707:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575985.046708:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575985.046709:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575985.046710:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575985.046712:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575985.046713:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575985.046718:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046721:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575985.046721:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046722:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575985.046723:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575985.046725:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046728:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575985.046730:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046734:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575985.046736:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046737:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575985.046738:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046739:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575985.046740:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575985.046741:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575985.046743:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575985.046744:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575985.046746:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575985.046754:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250688:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575985.046756:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046757:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575985.046759:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393250560/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046762:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575985.046763:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575985.046764:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788575985.046766:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575985.046766:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575985.046767:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575985.046768:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575985.046769:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788575985.046771:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575985.046771:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575985.046773:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:2.0:1788575985.046774:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575985.046775:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046776:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575985.046778:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393250688/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576010 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575985.046781:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575985.046782:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575985.046783:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788575985.046784:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575985.046784:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575985.046785:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575985.046786:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575985.046787:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:2.0:1788575985.046788:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575985.046788:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575985.046789:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:2.0:1788575985.046791:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575985.046791:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046792:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.046795:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575985.046795:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575985.046797:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575985.046798:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575985.429535:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575985.429540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575985.429546:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575985.429547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575985.429550:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575985.429552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575985.429716:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575985.429718:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575985.429720:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575985.557472:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575985.557479:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575985.557481:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575985.557486:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575985.557489:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575985.557492:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.557494:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575985.557497:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575985.557499:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575986.005480:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307589561, at 4307589568
00000100:00000010:0.0:1788575986.005494:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cba0.
00000100:00000040:1.0:1788575986.005503:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307589561, at 4307589568
00000100:00000010:1.0:1788575986.005513:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cc60.
00010000:00000001:3.0:1788575986.453569:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575986.453576:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575986.453581:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575986.453582:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575986.453587:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575986.453588:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575986.453693:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575986.453696:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575986.453699:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575986.581529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575986.581537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575986.581539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575986.581544:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575986.581549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575986.581552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575986.581555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575986.581557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575986.581560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575987.477522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575987.477536:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575987.477543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575987.477546:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575987.477548:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575987.477551:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575987.477554:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575987.477556:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575987.477559:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575987.477560:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575987.477564:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575987.477565:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575987.477567:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575987.477568:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575987.477569:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575987.477570:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575987.477572:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575987.477573:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575987.477636:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575987.477640:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575987.477643:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575987.605499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575987.605505:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575987.605506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575987.605510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575987.605513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575987.605517:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575987.605520:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575987.605522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575987.605525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575988.501519:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575988.501524:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575988.501527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575988.501528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575988.501530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575988.501530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575988.501598:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575988.501602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575988.501605:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575988.629504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575988.629510:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575988.629511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575988.629515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575988.629517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575988.629520:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575988.629522:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575988.629523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575988.629525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575989.525512:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575989.525520:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575989.525521:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575989.525525:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575989.525528:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575989.525530:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575989.525532:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575989.525534:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575989.525538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575989.525539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575989.525543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575989.525544:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575989.525607:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575989.525609:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575989.525615:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575989.653501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575989.653507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575989.653509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575989.653514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575989.653518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575989.653521:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575989.653523:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575989.653525:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575989.653527:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575990.165509:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575990.165527:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788575990.165530:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575990.165534:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575990.165538:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575990.165545:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12921
00000400:00000200:2.0:1788575990.165564:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575990.165570:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575990.165573:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575990.165575:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165578:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575990.165579:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165581:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575990.165584:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575990.165586:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788575990.165587:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788575990.165590:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788575990.165591:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788575990.165593:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788575990.165595:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165596:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575990.165598:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165599:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575990.165601:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788575990.165604:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165605:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575990.165613:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575990.165615:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575990.165617:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575990.165619:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788575990.165621:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575990.165629:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02a00.
00000020:00000040:2.0:1788575990.165631:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788575990.165634:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788575990.165635:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575990.165638:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575990.165640:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575990.165643:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788575990.165648:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788575990.165653:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575990.165660:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788575990.165669:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575990.165673:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575990.165678:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cce910e4] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575990.165700:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575990.165703:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788575990.165712:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788575990.165714:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575990.165716:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575990.165719:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788575990.165722:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12921
00000400:00000200:2.0:1788575990.165727:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575990.165729:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788575990.165731:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788575990.165733:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165735:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575990.165737:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165738:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788575990.165740:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788575990.165741:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788575990.165742:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788575990.165744:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1788575990.165744:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788575990.165749:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788575990.165750:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1788575990.165751:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788575990.165755:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575990.165756:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788575990.165757:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575990.165760:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165761:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575990.165765:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000040:0.0:1788575990.165766:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788575990.165768:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165772:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575990.165775:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788575990.165780:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575990.165781:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1788575990.165784:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.165786:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575990.165786:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788575990.165791:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788575990.165792:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:2.0:1788575990.165794:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575990.165797:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575990.165801:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575990.165805:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393250816:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575990.165806:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.165812:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:2.0:1788575990.165813:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03800.
00000020:00000040:2.0:1788575990.165817:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788575990.165818:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575990.165820:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575990.165822:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575990.165823:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575990.165826:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788575990.165827:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:0.0:1788575990.165833:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788575990.165836:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575990.165843:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575990.165847:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575990.165853:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
02000000:00000001:2.0:1788575990.165854:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575990.165858:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788575990.165860:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393250816, portal 25
00000100:00000001:0.0:1788575990.165864:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575990.165866:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:0.0:1788575990.165872:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788575990.165872:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000100:00000001:2.0:1788575990.165877:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575990.165885:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575990.165886:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575990.165894:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165895:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788575990.165899:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788575990.165901:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575990.165905:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393250816, offset 0
00000100:00000040:2.0:1788575990.165909:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014d98894] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788575990.165915:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165922:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788575990.165924:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788575990.165928:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788575990.165929:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788575990.165932:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788575990.165934:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788575990.165936:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:0.0:1788575990.165940:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1788575990.165940:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12921
00000400:00000200:0.0:1788575990.165950:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575990.165953:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575990.165958:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1788575990.165959:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1788575990.165961:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1788575990.165966:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1788575990.165967:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165972:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788575990.165973:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:2.0:1788575990.165977:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.165981:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788575990.165982:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331215:331215:256:4294967295] 192.168.202.110@tcp LPNI seq info [331172:331172:8:4294967295]
00000020:00000001:2.0:1788575990.165985:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1788575990.165990:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575990.165999:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1788575990.166003:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788575990.166006:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:0.0:1788575990.166010:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:2.0:1788575990.166010:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788575990.166014:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788575990.166017:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788575990.166023:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000010:0.0:1788575990.166024:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000100:00000001:2.0:1788575990.166025:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788575990.166029:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575990.166031:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788575990.166033:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:0.0:1788575990.166035:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575990.166038:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788575990.166042:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575990.166046:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00080000:2.0:1788575990.166047:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788575990.166052:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1788575990.166055:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788575990.166056:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:0.0:1788575990.166060:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
02000000:00000001:2.0:1788575990.166061:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575990.166064:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575990.166065:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1788575990.166067:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00100000:2.0:1788575990.166067:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788575990.166072:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575990.166075:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575990.166076:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000010:2.0:1788575990.166079:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00a80.
00000400:00000200:0.0:1788575990.166081:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000020:00000040:2.0:1788575990.166082:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000010:0.0:1788575990.166085:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575990.166089:0:2361:0:(events.c:35:request_out_callback()) Process entered
02000000:00000001:2.0:1788575990.166092:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:0.0:1788575990.166094:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788575990.166095:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575990.166099:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788575990.166101:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788575990.166103:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.166109:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575990.166112:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576015 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788575990.166114:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788575990.166119:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.166120:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.166125:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00100000:2.0:1788575990.166128:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575990.166129:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575990.166137:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575990.166139:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307594681
00000100:00000010:0.0:1788575990.166143:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:2.0:1788575990.166146:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575990.166148:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788575990.166149:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575990.166154:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000389a8694] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166155:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307594681
00000100:00000010:0.0:1788575990.166160:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7180.
00000100:00000001:0.0:1788575990.166164:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788575990.166166:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575990.166170:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575990.166170:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575990.166174:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1788575990.166175:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1788575990.166179:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00100000:0.0:1788575990.166180:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575990/real 1788575990]  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788575990.166184:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12931)
00000400:00000001:0.0:1788575990.166190:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575990.166192:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575990.166193:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575990.166195:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575990.166196:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575990.166197:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575990.166199:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575990.166225:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166230:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575990.166232:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166233:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575990.166235:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575990.166238:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166242:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575990.166246:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166251:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575990.166253:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166254:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575990.166255:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166256:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575990.166257:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575990.166259:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575990.166262:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575990.166263:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575990.166265:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575990.166270:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393250816:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788575990.166273:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166275:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575990.166276:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393250816/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166280:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575990.166281:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575990.166283:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788575990.166285:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575990.166287:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575990.166288:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788575990.166289:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575990.166291:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788575990.166293:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575990.166294:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575990.166295:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:0.0:1788575990.166297:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575990.166298:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166300:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166305:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575990.166306:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.166307:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575990.166308:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575990.166310:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575990.166312:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166316:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575990.166317:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575990.166318:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575990.166319:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575990.166320:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575990.166322:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250944:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575990.166324:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575990.166325:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575990.166326:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575990.166327:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575990.166329:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788575990.166330:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575990.166332:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575990.166334:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575990.166336:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575990.166338:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393250944, portal 10
00000100:00000001:0.0:1788575990.166340:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575990.166340:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:0.0:1788575990.166344:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166348:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575990.166349:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575990.166351:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575990.166353:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393250944, offset 0
00000400:00000200:0.0:1788575990.166355:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575990.166357:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575990.166359:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575990.166361:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575990.166363:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575990.166365:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575990.166366:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575990.166368:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331216:331216:256:4294967295] 192.168.202.110@tcp LPNI seq info [331173:331173:8:4294967295]
00000400:00000200:0.0:1788575990.166370:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575990.166374:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575990.166376:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575990.166379:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575990.166381:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575990.166383:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788575990.166385:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575990.166387:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788575990.166389:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575990.166406:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575990.166408:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575990.166410:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575990.166412:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575990.166413:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575990.166415:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575990.166417:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166432:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575990.166433:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576015 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166437:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.166438:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575990.166440:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575990.166442:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788575990.166444:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575990.166446:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307593767
00000100:00000001:0.0:1788575990.166448:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307594681
00000100:00000001:0.0:1788575990.166449:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307593767
00000100:00000001:0.0:1788575990.166450:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307594681
00000100:00000001:0.0:1788575990.166451:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575990.166453:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.166454:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575990.166456:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575990/real 1788575990]  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575990.166460:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575990.166461:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575990.166462:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575990.166463:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575990.166464:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575990.166465:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575990.166467:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575990.166472:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166475:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575990.166476:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166478:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575990.166479:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575990.166483:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166489:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575990.166493:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166499:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575990.166501:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166502:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575990.166504:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166505:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575990.166507:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575990.166510:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575990.166513:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575990.166515:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575990.166519:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575990.166527:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393250944:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788575990.166530:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575990.166534:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166540:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575990.166541:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575990.166543:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788575990.166545:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575990.166547:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575990.166550:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251072:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575990.166553:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788575990.166555:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788575990.166557:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575990.166558:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788575990.166562:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575990.166564:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788575990.166568:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788575990.166570:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788575990.166573:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788575990.166576:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393251072, portal 4
00000100:00000001:0.0:1788575990.166578:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788575990.166579:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:0.0:1788575990.166584:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166588:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788575990.166590:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788575990.166591:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788575990.166592:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393251072, offset 0
00000400:00000200:0.0:1788575990.166594:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788575990.166597:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575990.166600:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788575990.166602:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788575990.166603:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788575990.166605:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788575990.166606:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788575990.166609:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331217:331217:256:4294967295] 192.168.202.110@tcp LPNI seq info [331174:331174:8:4294967295]
00000400:00000200:0.0:1788575990.166613:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575990.166619:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788575990.166623:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575990.166626:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788575990.166630:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788575990.166633:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788575990.166636:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788575990.166639:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788575990.166643:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788575990.166644:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788575990.166647:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788575990.166651:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788575990.166653:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788575990.166655:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788575990.166657:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788575990.166660:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576015 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166665:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575990.166666:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576015 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166670:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.166671:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788575990.166673:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575990.166675:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788575990.166677:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575990.166678:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307593913
00000100:00000001:0.0:1788575990.166680:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307594681
00000100:00000001:0.0:1788575990.166681:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307593913
00000100:00000001:0.0:1788575990.166683:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307594681
00000100:00000001:0.0:1788575990.166684:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788575990.166685:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.166686:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788575990.166688:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575990/real 1788575990]  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788575990.166692:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788575990.166694:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788575990.166695:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788575990.166696:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788575990.166697:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788575990.166698:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788575990.166700:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788575990.166704:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166708:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788575990.166709:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166710:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575990.166711:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788575990.166713:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166716:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788575990.166718:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166722:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788575990.166723:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166724:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788575990.166725:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166726:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788575990.166727:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788575990.166728:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788575990.166730:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788575990.166731:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788575990.166733:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575990.166741:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251072:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788575990.166743:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166744:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575990.166746:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393250944/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166749:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575990.166750:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575990.166751:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788575990.166752:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575990.166753:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575990.166754:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788575990.166755:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575990.166756:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788575990.166757:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575990.166758:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575990.166759:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:0.0:1788575990.166761:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575990.166762:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166763:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788575990.166764:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393251072/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576015 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575990.166768:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788575990.166769:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788575990.166769:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788575990.166770:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788575990.166771:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788575990.166772:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788575990.166772:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788575990.166773:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788575990.166774:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788575990.166775:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788575990.166776:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:0.0:1788575990.166777:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788575990.166778:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166779:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575990.166782:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575990.166782:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575990.166784:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788575990.166785:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575990.549525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575990.549531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575990.549537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575990.549538:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575990.549541:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575990.549543:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575990.549594:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575990.549596:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575990.549599:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575990.677501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575990.677506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575990.677507:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575990.677511:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575990.677513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575990.677516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575990.677518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575990.677520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575990.677522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575991.125520:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307594681, at 4307594688
00000100:00000040:1.0:1788575991.125520:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307594681, at 4307594688
00000100:00000010:1.0:1788575991.125535:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7180.
00000100:00000010:0.0:1788575991.125538:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00010000:00000001:3.0:1788575991.573534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575991.573540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575991.573546:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575991.573547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575991.573550:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575991.573551:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575991.573657:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575991.573661:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575991.573664:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575991.701480:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575991.701486:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575991.701487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575991.701491:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575991.701494:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575991.701496:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575991.701498:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575991.701500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575991.701502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575992.597519:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575992.597524:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575992.597527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575992.597528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575992.597530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575992.597531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575992.597586:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575992.597590:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575992.597593:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1788575992.725469:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1788575992.725474:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575992.725476:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575992.725480:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:2.0:1788575992.725482:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575992.725485:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575992.725487:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575992.725488:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575992.725490:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575993.621535:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575993.621541:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575993.621546:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575993.621547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575993.621550:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575993.621551:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575993.621608:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575993.621612:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575993.621615:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575993.749535:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575993.749542:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575993.749544:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575993.749551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575993.749555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575993.749559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575993.749562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575993.749564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575993.749567:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575994.645505:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575994.645511:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575994.645517:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575994.645519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575994.645522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575994.645529:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575994.645585:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575994.645589:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575994.645592:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575994.773544:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575994.773552:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575994.773555:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575994.773560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575994.773563:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575994.773568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575994.773570:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575994.773573:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575994.773576:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575995.285523:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575995.285531:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788575995.285534:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575995.285537:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575995.285541:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575995.285547:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12926
00000400:00000200:0.0:1788575995.285556:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788575995.285561:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788575995.285565:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788575995.285567:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575995.285570:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575995.285572:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575995.285574:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575995.285578:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575995.285580:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788575995.285583:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788575995.285585:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788575995.285587:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788575995.285589:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788575995.285592:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575995.285593:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575995.285595:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575995.285597:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575995.285599:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788575995.285602:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788575995.285605:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575995.285607:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788575995.285610:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575995.285612:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575995.285614:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788575995.285616:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788575995.285624:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788575995.285627:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788575995.285630:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575995.285631:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575995.285634:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575995.285636:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575995.285649:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575995.285657:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788575995.285662:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575995.285670:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788575995.285679:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788575995.285682:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575995.285686:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575995.285706:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788575995.285709:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788575995.285712:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788575995.285717:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788575995.285718:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788575995.285721:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788575995.285724:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12926
00000400:00000200:0.0:1788575995.285728:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575995.285730:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1788575995.285733:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1788575995.285734:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1788575995.285737:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1788575995.285738:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788575995.285741:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:2.0:1788575995.285745:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575995.285746:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788575995.285751:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575995.285756:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575995.285761:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788575995.285762:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1788575995.285764:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:2.0:1788575995.285765:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575995.285767:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1788575995.285769:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1788575995.285771:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
02000000:00000001:2.0:1788575995.285772:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1788575995.285775:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:2.0:1788575995.285775:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575995.285780:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00100000:2.0:1788575995.285781:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393251200:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575995.285787:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:0.0:1788575995.285788:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575995.285791:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1788575995.285791:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575995.285794:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575995.285795:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:2.0:1788575995.285796:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788575995.285798:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788575995.285801:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000010:2.0:1788575995.285802:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
00000100:00080000:0.0:1788575995.285805:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788575995.285805:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575995.285810:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575995.285813:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1788575995.285815:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575995.285818:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
02000000:00000001:0.0:1788575995.285819:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788575995.285822:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575995.285825:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575995.285828:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000010:2.0:1788575995.285830:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000008:00000001:0.0:1788575995.285831:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575995.285835:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575995.285837:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393251200, portal 25
00000100:00000010:0.0:1788575995.285839:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000100:00000001:2.0:1788575995.285841:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000020:00000040:0.0:1788575995.285844:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788575995.285844:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
02000000:00000001:0.0:1788575995.285851:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:2.0:1788575995.285853:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1788575995.285854:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575995.285858:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575995.285860:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788575995.285863:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.285864:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575995.285867:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000010:0.0:1788575995.285868:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000400:00000010:2.0:1788575995.285870:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000001:0.0:1788575995.285873:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788575995.285875:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393251200, offset 0
00000400:00000200:2.0:1788575995.285882:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00100000:0.0:1788575995.285883:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788575995.285889:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575995.285895:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575995.285898:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788575995.285899:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575995.285904:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1788575995.285908:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788575995.285910:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575995.285915:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575995.285919:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575995.285925:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331218:331218:256:4294967295] 192.168.202.110@tcp LPNI seq info [331175:331175:8:4294967295]
00000100:00000001:0.0:1788575995.285929:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575995.285930:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788575995.285935:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788575995.285943:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575995.285947:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1788575995.285951:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000010:2.0:1788575995.285951:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000100:00000001:0.0:1788575995.285955:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:2.0:1788575995.285959:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1788575995.285960:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:2.0:1788575995.285963:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:0.0:1788575995.285965:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000800:00000200:2.1:1788575995.285971:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1788575995.285972:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12926
00000800:00000200:2.1:1788575995.285974:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000400:00000200:0.0:1788575995.285979:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000100:2.0:1788575995.285980:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575995.285984:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:0.0:1788575995.285987:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788575995.285989:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1788575995.285991:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788575995.285995:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1788575995.285999:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788575995.286000:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000100:00000001:0.0:1788575995.286004:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1788575995.286004:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575995.286009:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:0.0:1788575995.286010:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000200:2.0:1788575995.286015:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575995.286016:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788575995.286021:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788575995.286028:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:2.0:1788575995.286030:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000020:00000001:0.0:1788575995.286032:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1788575995.286036:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576020 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1788575995.286037:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788575995.286042:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788575995.286047:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788575995.286054:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788575995.286055:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575995.286057:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788575995.286059:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:0.0:1788575995.286062:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788575995.286065:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788575995.286068:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1788575995.286069:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788575995.286075:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1788575995.286077:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788575995.286079:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788575995.286083:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788575995.286083:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307599801
02000000:00000001:0.0:1788575995.286088:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:2.0:1788575995.286091:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c840.
02000000:00000001:0.0:1788575995.286092:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575995.286101:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788575995.286103:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788575995.286106:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286107:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788575995.286110:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286115:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307599801
00000100:00000010:0.0:1788575995.286116:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788575995.286119:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788575995.286122:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788575995.286123:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788575995.286123:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c000.
00000100:00000001:0.0:1788575995.286127:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788575995.286129:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788575995.286130:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:0.0:1788575995.286131:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788575995.286137:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:2.0:1788575995.286137:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575995.286141:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788575995.286144:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286144:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575995.286150:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575995/real 1788575995]  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788575995.286154:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:2.0:1788575995.286160:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575995.286163:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575995.286166:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000100:00000001:0.0:1788575995.286169:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1788575995.286169:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575995.286172:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:0.0:1788575995.286174:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286175:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000040:0.0:1788575995.286184:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575995.286184:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575995.286205:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788575995.286208:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575995.286212:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788575995.286215:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:0.0:1788575995.286216:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788575995.286218:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286221:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:0.0:1788575995.286222:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12936)
00000100:00000001:2.0:1788575995.286222:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575995.286225:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286231:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575995.286234:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286238:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575995.286240:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286241:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575995.286242:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286243:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575995.286244:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575995.286246:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575995.286249:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575995.286250:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575995.286252:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575995.286257:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393251200:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788575995.286260:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286261:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575995.286263:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393251200/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286266:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575995.286268:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575995.286269:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575995.286271:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575995.286272:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575995.286273:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788575995.286274:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575995.286276:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:2.0:1788575995.286278:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575995.286279:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575995.286280:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:2.0:1788575995.286282:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575995.286283:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286285:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286290:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575995.286291:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286292:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575995.286293:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788575995.286294:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575995.286296:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286301:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575995.286302:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575995.286303:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575995.286304:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575995.286306:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575995.286307:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575995.286309:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575995.286310:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575995.286311:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575995.286312:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575995.286314:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575995.286315:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575995.286317:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575995.286319:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575995.286321:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575995.286323:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393251328, portal 10
00000100:00000001:2.0:1788575995.286324:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575995.286325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000040:2.0:1788575995.286329:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286333:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575995.286334:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575995.286336:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575995.286337:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393251328, offset 0
00000400:00000200:2.0:1788575995.286339:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575995.286341:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575995.286344:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575995.286345:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575995.286347:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575995.286348:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575995.286350:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575995.286352:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331219:331219:256:4294967295] 192.168.202.110@tcp LPNI seq info [331176:331176:8:4294967295]
00000400:00000200:2.0:1788575995.286354:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575995.286358:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575995.286361:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575995.286363:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788575995.286387:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575995.286389:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788575995.286406:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575995.286408:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1788575995.286412:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575995.286413:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788575995.286416:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575995.286420:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575995.286423:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575995.286425:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575995.286427:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575995.286431:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286438:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575995.286441:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576020 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286448:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286449:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575995.286452:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575995.286456:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788575995.286459:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575995.286461:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307598887
00000100:00000001:2.0:1788575995.286463:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307599801
00000100:00000001:2.0:1788575995.286465:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307598887
00000100:00000001:2.0:1788575995.286466:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307599801
00000100:00000001:2.0:1788575995.286467:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575995.286469:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286470:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575995.286472:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575995/real 1788575995]  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575995.286481:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575995.286482:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575995.286484:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575995.286486:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575995.286487:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575995.286489:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575995.286493:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575995.286510:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286516:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575995.286517:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286519:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575995.286519:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575995.286522:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286526:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575995.286528:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286532:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575995.286534:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286534:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575995.286535:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286536:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575995.286537:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575995.286539:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575995.286540:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575995.286541:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575995.286544:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575995.286549:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788575995.286551:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788575995.286553:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286556:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788575995.286558:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575995.286559:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788575995.286560:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788575995.286565:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788575995.286567:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575995.286568:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788575995.286570:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788575995.286571:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788575995.286572:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788575995.286574:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575995.286576:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788575995.286578:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788575995.286580:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788575995.286582:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788575995.286584:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393251456, portal 4
00000100:00000001:2.0:1788575995.286585:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788575995.286586:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:2.0:1788575995.286590:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286594:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788575995.286595:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788575995.286596:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788575995.286598:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393251456, offset 0
00000400:00000200:2.0:1788575995.286601:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788575995.286603:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575995.286605:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788575995.286607:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788575995.286609:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788575995.286610:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788575995.286611:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788575995.286613:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331220:331220:256:4294967295] 192.168.202.110@tcp LPNI seq info [331177:331177:8:4294967295]
00000400:00000200:2.0:1788575995.286616:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788575995.286620:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788575995.286622:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575995.286624:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788575995.286626:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788575995.286627:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788575995.286630:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788575995.286631:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1788575995.286634:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788575995.286635:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788575995.286637:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788575995.286640:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788575995.286643:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788575995.286644:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788575995.286646:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788575995.286649:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576020 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286656:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575995.286659:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576020 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286664:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286666:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788575995.286669:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788575995.286672:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788575995.286675:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788575995.286678:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307599033
00000100:00000001:2.0:1788575995.286680:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307599801
00000100:00000001:2.0:1788575995.286682:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307599033
00000100:00000001:2.0:1788575995.286684:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307599801
00000100:00000001:2.0:1788575995.286686:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788575995.286688:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286690:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788575995.286693:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575995/real 1788575995]  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788575995.286698:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788575995.286699:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788575995.286700:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788575995.286702:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788575995.286703:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788575995.286704:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788575995.286706:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788575995.286710:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286714:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788575995.286715:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286715:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575995.286716:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788575995.286718:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286722:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788575995.286724:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286728:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788575995.286730:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788575995.286731:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286732:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788575995.286734:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788575995.286735:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788575995.286737:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788575995.286738:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788575995.286740:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788575995.286744:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788575995.286747:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286748:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575995.286750:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393251328/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286753:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575995.286754:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575995.286755:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788575995.286756:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575995.286757:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575995.286758:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788575995.286759:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575995.286760:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:2.0:1788575995.286762:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575995.286762:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575995.286763:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:2.0:1788575995.286765:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575995.286766:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286767:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788575995.286768:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393251456/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576020 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788575995.286776:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788575995.286776:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788575995.286777:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788575995.286779:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788575995.286779:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788575995.286780:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788575995.286781:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788575995.286782:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788575995.286783:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788575995.286783:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788575995.286784:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:2.0:1788575995.286785:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788575995.286786:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286787:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788575995.286789:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788575995.286790:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788575995.286791:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788575995.286791:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575995.669485:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575995.669491:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575995.669494:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575995.669495:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575995.669498:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575995.669498:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575995.669549:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575995.669552:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575995.669555:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575995.797493:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575995.797498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575995.797501:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575995.797506:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575995.797510:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575995.797513:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575995.797516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575995.797520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575995.797522:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788575996.245500:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307599801, at 4307599808
00000100:00000040:1.0:1788575996.245500:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307599801, at 4307599808
00000100:00000010:0.0:1788575996.245515:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c840.
00000100:00000010:1.0:1788575996.245517:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c000.
00010000:00000001:3.0:1788575996.693498:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575996.693505:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575996.693510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575996.693512:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575996.693516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575996.693518:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575996.693582:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575996.693586:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575996.693589:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575996.821509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575996.821515:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575996.821517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575996.821523:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575996.821527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575996.821530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575996.821533:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575996.821534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575996.821536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575997.718091:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575997.718097:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575997.718102:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575997.718105:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575997.718107:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575997.718110:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575997.718113:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575997.718115:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575997.718117:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575997.718119:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575997.718122:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575997.718123:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575997.718125:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575997.718126:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575997.718128:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575997.718129:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575997.718131:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575997.718132:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575997.718195:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575997.718199:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575997.718202:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575997.845520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575997.845526:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575997.845527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575997.845531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575997.845533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575997.845536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575997.845538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575997.845539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575997.845541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575998.741455:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575998.741462:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788575998.741464:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788575998.741467:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575998.741482:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788575998.741484:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575998.741487:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575998.741489:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575998.741493:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575998.741494:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575998.741498:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575998.741499:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788575998.741613:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788575998.741616:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788575998.741619:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575998.869506:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575998.869514:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575998.869517:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575998.869521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575998.869525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575998.869529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575998.869531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575998.869534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575998.869536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575999.765539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575999.765545:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575999.765551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575999.765552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788575999.765556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788575999.765557:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788575999.765619:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788575999.765623:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788575999.765626:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788575999.893571:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788575999.893577:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788575999.893578:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788575999.893583:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788575999.893586:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788575999.893589:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575999.893592:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788575999.893595:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788575999.893598:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576000.405502:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576000.405512:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576000.405515:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576000.405519:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576000.405536:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576000.405543:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12931
00000400:00000200:3.0:1788576000.405552:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576000.405557:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576000.405566:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576000.405569:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405571:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576000.405574:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405576:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576000.405580:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576000.405582:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576000.405584:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576000.405587:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576000.405590:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576000.405592:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576000.405595:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405597:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576000.405599:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405601:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576000.405604:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576000.405607:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405610:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576000.405613:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576000.405615:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576000.405618:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576000.405620:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576000.405623:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576000.405632:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0a80.
00000020:00000040:3.0:1788576000.405636:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576000.405640:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576000.405641:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576000.405644:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576000.405647:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576000.405651:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576000.405658:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa000.
00000100:00000001:3.0:1788576000.405665:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576000.405675:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576000.405685:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576000.405689:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576000.405694:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b298c26f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576000.405725:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576000.405728:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576000.405731:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576000.405732:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576000.405734:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576000.405736:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576000.405739:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12931
00000400:00000200:3.0:1788576000.405743:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576000.405747:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576000.405749:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576000.405753:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405755:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576000.405758:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405760:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788576000.405762:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788576000.405763:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576000.405765:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576000.405768:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788576000.405769:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1788576000.405771:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:2.0:1788576000.405773:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788576000.405774:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576000.405778:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788576000.405783:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788576000.405784:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405788:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576000.405792:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405797:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576000.405803:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788576000.405804:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576000.405808:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576000.405808:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405813:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576000.405814:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576000.405816:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576000.405818:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576000.405822:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576000.405823:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576000.405827:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576000.405829:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393251584:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788576000.405830:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576000.405835:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788576000.405836:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:3.0:1788576000.405840:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576000.405842:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1788576000.405843:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576000.405847:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576000.405849:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2300.
02000000:00000001:2.0:1788576000.405852:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1788576000.405853:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788576000.405857:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576000.405859:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576000.405860:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
00000100:00000001:3.0:1788576000.405861:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576000.405864:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576000.405865:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576000.405867:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576000.405873:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1788576000.405873:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
00000100:00000001:3.0:1788576000.405879:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788576000.405880:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00100000:3.0:1788576000.405886:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1788576000.405888:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576000.405897:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393251584, portal 25
00000100:00000001:3.0:1788576000.405897:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576000.405901:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.405902:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576000.405906:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:3.0:1788576000.405908:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc4bce75] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788576000.405915:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.405925:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576000.405930:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405930:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788576000.405935:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00080000:3.0:1788576000.405935:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576000.405941:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000200:2.0:1788576000.405942:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393251584, offset 0
00000100:00000001:3.0:1788576000.405944:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576000.405947:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788576000.405952:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576000.405953:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576000.405958:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12931
00000400:00000200:2.0:1788576000.405963:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576000.405971:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576000.405976:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576000.405980:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576000.405984:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576000.405987:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576000.405991:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331221:331221:256:4294967295] 192.168.202.110@tcp LPNI seq info [331178:331178:8:4294967295]
00000400:00000200:2.0:1788576000.405996:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576000.406002:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576000.406011:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788576000.406012:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1788576000.406015:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576000.406021:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:2.0:1788576000.406024:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576000.406025:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576000.406029:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:2.0:1788576000.406031:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000100:00000001:3.0:1788576000.406033:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000800:00000200:2.0:1788576000.406041:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576000.406045:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:3.0:1788576000.406046:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576000.406049:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576000.406052:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576000.406056:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000200:2.1:1788576000.406056:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1788576000.406060:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:2.1:1788576000.406062:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:3.0:1788576000.406064:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788576000.406072:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000100:2.0:1788576000.406074:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576000.406075:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000010:2.0:1788576000.406078:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000100:00000040:3.0:1788576000.406080:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576000.406082:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576000.406085:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788576000.406087:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576000.406091:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576000.406094:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576000.406098:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788576000.406100:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:3.0:1788576000.406101:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576000.406107:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1788576000.406113:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1788576000.406115:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000008:00100000:3.0:1788576000.406116:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000010:2.0:1788576000.406120:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000008:00000001:3.0:1788576000.406121:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576000.406125:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.406126:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000010:3.0:1788576000.406131:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000100:00000200:2.0:1788576000.406132:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788576000.406135:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788576000.406140:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576000.406143:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576000.406147:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576000.406150:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1788576000.406153:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1788576000.406153:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576000.406159:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576025 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788576000.406160:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788576000.406165:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.406173:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576000.406174:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576000.406178:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576000.406184:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576000.406186:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576000.406190:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576000.406191:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788576000.406196:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406197:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307604921
00000100:00000010:2.0:1788576000.406205:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c600.
00000100:00000001:3.0:1788576000.406209:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576000.406213:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1788576000.406214:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:3.0:1788576000.406219:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788576000.406223:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307604921
00000100:00000040:3.0:1788576000.406225:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12941)
00000100:00000010:2.0:1788576000.406229:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cf00.
00000100:00000001:2.0:1788576000.406234:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576000.406237:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576000.406240:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.406243:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576000.406248:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576000/real 1788576000]  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576000.406258:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576000.406260:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576000.406262:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576000.406265:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576000.406266:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576000.406269:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576000.406272:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576000.406280:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406287:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576000.406289:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406291:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576000.406293:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576000.406297:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406303:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576000.406307:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406314:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576000.406317:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406318:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576000.406319:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406321:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576000.406323:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576000.406326:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576000.406329:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576000.406331:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576000.406335:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576000.406342:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393251584:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576000.406346:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406348:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576000.406351:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393251584/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406356:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576000.406359:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576000.406362:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788576000.406365:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576000.406366:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576000.406368:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576000.406370:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576000.406372:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:2.0:1788576000.406375:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576000.406377:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576000.406378:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:2.0:1788576000.406381:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576000.406382:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406384:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406417:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576000.406419:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.406421:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576000.406422:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576000.406424:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576000.406427:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406433:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576000.406435:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576000.406436:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576000.406438:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576000.406440:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576000.406443:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251712:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576000.406445:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576000.406447:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576000.406449:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576000.406450:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576000.406453:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788576000.406455:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576000.406458:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576000.406461:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576000.406464:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576000.406467:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393251712, portal 10
00000100:00000001:2.0:1788576000.406470:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576000.406472:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000040:2.0:1788576000.406478:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406484:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576000.406486:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576000.406489:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576000.406491:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393251712, offset 0
00000400:00000200:2.0:1788576000.406495:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576000.406499:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576000.406502:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576000.406505:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576000.406508:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576000.406510:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576000.406512:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576000.406515:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331222:331222:256:4294967295] 192.168.202.110@tcp LPNI seq info [331179:331179:8:4294967295]
00000400:00000200:2.0:1788576000.406519:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576000.406534:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576000.406539:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576000.406543:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576000.406546:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576000.406549:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788576000.406553:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576000.406556:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1788576000.406559:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576000.406561:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576000.406564:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576000.406567:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576000.406570:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576000.406572:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576000.406574:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576000.406577:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406590:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576000.406593:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576025 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406599:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.406600:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576000.406603:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576000.406607:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576000.406610:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576000.406612:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307604007
00000100:00000001:2.0:1788576000.406615:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307604921
00000100:00000001:2.0:1788576000.406617:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307604007
00000100:00000001:2.0:1788576000.406619:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307604921
00000100:00000001:2.0:1788576000.406621:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576000.406623:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.406625:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576000.406628:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576000/real 1788576000]  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576000.406635:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576000.406637:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576000.406638:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576000.406640:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576000.406642:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576000.406644:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576000.406647:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576000.406654:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406668:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576000.406670:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406672:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576000.406676:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576000.406680:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406686:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576000.406690:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406696:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576000.406699:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406700:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576000.406701:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406703:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576000.406705:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576000.406708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576000.406710:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576000.406713:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576000.406717:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576000.406724:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251712:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576000.406728:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576000.406731:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406737:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576000.406739:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576000.406741:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576000.406742:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576000.406744:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576000.406747:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251840:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576000.406750:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576000.406753:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576000.406754:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576000.406756:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576000.406759:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788576000.406761:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576000.406764:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576000.406766:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576000.406769:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576000.406772:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393251840, portal 4
00000100:00000001:2.0:1788576000.406774:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576000.406775:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788576000.406780:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406786:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576000.406788:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576000.406790:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576000.406792:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393251840, offset 0
00000400:00000200:2.0:1788576000.406796:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576000.406799:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576000.406803:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576000.406805:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576000.406808:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576000.406810:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576000.406812:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576000.406815:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331223:331223:256:4294967295] 192.168.202.110@tcp LPNI seq info [331180:331180:8:4294967295]
00000400:00000200:2.0:1788576000.406819:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576000.406825:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576000.406829:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576000.406832:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576000.406836:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576000.406839:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788576000.406842:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576000.406845:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1788576000.406848:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576000.406850:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576000.406852:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576000.406856:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576000.406858:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576000.406860:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576000.406862:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576000.406865:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576025 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406873:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576000.406875:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576025 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406881:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.406883:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576000.406885:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576000.406888:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576000.406891:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576000.406893:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307604153
00000100:00000001:2.0:1788576000.406895:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307604921
00000100:00000001:2.0:1788576000.406897:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307604153
00000100:00000001:2.0:1788576000.406899:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307604921
00000100:00000001:2.0:1788576000.406901:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576000.406903:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.406905:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576000.406908:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576000/real 1788576000]  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576000.406916:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576000.406918:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576000.406919:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576000.406921:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576000.406923:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576000.406925:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576000.406927:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576000.406932:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406936:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576000.406937:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406938:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576000.406939:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576000.406940:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406944:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576000.406946:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406950:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576000.406952:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406953:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576000.406953:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406954:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576000.406955:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576000.406957:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576000.406958:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576000.406959:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576000.406962:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576000.406969:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393251840:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576000.406971:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406973:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576000.406974:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393251712/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406978:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576000.406979:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576000.406980:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788576000.406982:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576000.406983:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576000.406983:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576000.406985:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576000.406986:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:2.0:1788576000.406987:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576000.406988:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576000.406989:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:2.0:1788576000.406991:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576000.406992:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.406993:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576000.406994:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393251840/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576025 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576000.406998:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576000.406998:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576000.406999:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788576000.407000:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576000.407001:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576000.407001:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576000.407002:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576000.407003:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788576000.407004:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576000.407004:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576000.407005:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788576000.407007:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576000.407008:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.407009:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576000.407011:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576000.407012:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576000.407013:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576000.407014:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576000.790112:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576000.790118:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576000.790123:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576000.790125:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576000.790128:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576000.790129:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576000.790194:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576000.790197:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576000.790199:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576000.917500:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576000.917507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576000.917508:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576000.917514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576000.917517:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576000.917522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576000.917524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576000.917527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576000.917530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576001.365474:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307604921, at 4307604928
00000100:00000040:0.0:1788576001.365487:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307604921, at 4307604928
00000100:00000010:1.0:1788576001.365488:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cf00.
00000100:00000010:0.0:1788576001.365497:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c600.
00010000:00000001:3.0:1788576001.813497:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576001.813502:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576001.813508:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576001.813509:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576001.813513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576001.813515:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576001.813653:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576001.813656:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576001.813659:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576001.941522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576001.941531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576001.941533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576001.941539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576001.941543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576001.941547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576001.941550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576001.941552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576001.941555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576002.837506:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576002.837511:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576002.837514:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576002.837515:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576002.837518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576002.837519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576002.837586:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576002.837590:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576002.837593:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576002.965511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576002.965520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576002.965522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576002.965528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576002.965532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576002.965536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576002.965538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576002.965540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576002.965543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576003.861487:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576003.861493:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576003.861498:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576003.861500:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576003.861503:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576003.861504:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576003.861547:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576003.861550:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576003.861552:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576003.989509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576003.989517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576003.989520:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576003.989525:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576003.989529:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576003.989533:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576003.989535:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576003.989537:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576003.989540:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576004.885521:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576004.885527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576004.885530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576004.885531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576004.885533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576004.885534:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576004.885589:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576004.885594:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576004.885597:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576005.013505:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576005.013512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576005.013514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576005.013520:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576005.013523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576005.013527:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576005.013530:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576005.013532:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576005.013535:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576005.525515:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576005.525528:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576005.525530:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576005.525534:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576005.525537:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576005.525543:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12936
00000400:00000200:3.0:1788576005.525550:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576005.525555:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576005.525557:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576005.525559:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525562:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576005.525564:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525566:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576005.525569:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576005.525570:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576005.525572:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576005.525575:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576005.525576:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576005.525578:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576005.525579:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525580:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576005.525581:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525582:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576005.525584:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576005.525586:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525588:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576005.525590:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576005.525592:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576005.525594:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576005.525595:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576005.525597:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576005.525604:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788576005.525607:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576005.525609:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576005.525611:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576005.525613:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576005.525615:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576005.525619:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576005.525625:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788576005.525630:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576005.525636:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576005.525643:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576005.525646:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576005.525650:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576005.525668:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576005.525670:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576005.525672:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576005.525673:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576005.525689:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576005.525693:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576005.525696:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12936
00000400:00000200:3.0:1788576005.525700:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576005.525704:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576005.525707:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576005.525710:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525712:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576005.525714:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576005.525716:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788576005.525717:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788576005.525722:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1788576005.525723:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788576005.525726:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788576005.525726:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576005.525730:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576005.525733:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788576005.525736:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788576005.525737:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1788576005.525739:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788576005.525747:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525753:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1788576005.525755:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1788576005.525758:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576005.525759:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576005.525763:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:2.0:1788576005.525766:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1788576005.525769:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788576005.525772:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525776:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576005.525778:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576005.525781:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576005.525786:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576005.525786:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576005.525793:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576005.525794:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393251968:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788576005.525796:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576005.525802:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1788576005.525803:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:3.0:1788576005.525808:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576005.525809:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1788576005.525811:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576005.525813:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576005.525817:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
02000000:00000001:2.0:1788576005.525818:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1788576005.525823:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000010:2.0:1788576005.525826:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:3.0:1788576005.525826:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576005.525829:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576005.525831:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576005.525833:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576005.525836:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576005.525838:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576005.525843:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1788576005.525846:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000400:00000010:2.0:1788576005.525849:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00000001:3.0:1788576005.525852:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788576005.525857:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00100000:3.0:1788576005.525861:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1788576005.525866:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393251968, portal 25
00000100:00000001:3.0:1788576005.525874:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576005.525877:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.525878:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1788576005.525883:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.525885:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788576005.525898:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576030 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576005.525898:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576005.525904:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576005.525912:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788576005.525916:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788576005.525917:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576005.525921:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1788576005.525922:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525926:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000010:2.0:1788576005.525928:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00080000:3.0:1788576005.525932:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12936
00000100:00000200:2.0:1788576005.525934:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393251968, offset 0
00000400:00000200:3.0:1788576005.525939:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576005.525945:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576005.525947:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576005.525951:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1788576005.525957:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788576005.525957:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525963:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788576005.525968:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1788576005.525968:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576005.525975:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788576005.525978:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:3.0:1788576005.525979:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576005.525983:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576005.525985:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576005.525987:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788576005.525988:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788576005.525991:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788576005.525993:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1788576005.525998:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:2.0:1788576005.526001:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1788576005.526002:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788576005.526007:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:3.0:1788576005.526009:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576005.526014:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:2.0:1788576005.526015:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331224:331224:256:4294967295] 192.168.202.110@tcp LPNI seq info [331181:331181:8:4294967295]
00000020:00000040:3.0:1788576005.526018:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:2.0:1788576005.526024:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1788576005.526025:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576005.526031:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576005.526034:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576005.526039:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1788576005.526040:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1788576005.526045:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576005.526050:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:2.0:1788576005.526051:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00100000:3.0:1788576005.526053:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000010:2.0:1788576005.526057:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000008:00000001:3.0:1788576005.526058:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576005.526062:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576005.526066:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000800:00000200:2.0:1788576005.526067:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1788576005.526071:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000800:00000200:2.0:1788576005.526074:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
02000000:00000001:3.0:1788576005.526076:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576005.526080:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576005.526085:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:2.1:1788576005.526085:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:3.0:1788576005.526089:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:2.1:1788576005.526091:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
02000000:00000001:3.0:1788576005.526093:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1788576005.526099:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000010:3.0:1788576005.526100:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000800:00000010:2.0:1788576005.526103:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000100:00000001:3.0:1788576005.526105:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576005.526111:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:3.0:1788576005.526114:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788576005.526121:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576005.526131:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000100:00000001:3.0:1788576005.526132:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1788576005.526137:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:3.0:1788576005.526137:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.526145:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:3.0:1788576005.526148:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576005.526153:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576030 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576005.526169:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.526174:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576005.526174:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788576005.526181:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576030 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788576005.526182:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788576005.526190:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12946)
00000100:00000001:2.0:1788576005.526193:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.526197:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576005.526202:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576005.526207:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576005.526217:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307610041
00000100:00000010:2.0:1788576005.526222:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c4e0.
00000100:00000001:2.0:1788576005.526229:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576005.526234:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307610041
00000100:00000010:2.0:1788576005.526237:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ccc0.
00000100:00000001:2.0:1788576005.526240:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576005.526244:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576005.526247:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.526249:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576005.526253:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576005/real 1788576005]  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576005.526263:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576005.526265:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576005.526267:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576005.526270:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576005.526272:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576005.526274:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576005.526278:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576005.526293:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526300:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576005.526302:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526305:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576005.526307:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576005.526311:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526317:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576005.526322:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526329:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576005.526333:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526334:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576005.526336:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526338:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576005.526340:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576005.526343:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576005.526346:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576005.526349:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576005.526353:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576005.526368:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393251968:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576005.526373:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526376:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576005.526379:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393251968/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526386:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576005.526388:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576005.526413:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788576005.526417:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576005.526419:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576005.526421:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576005.526423:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576005.526426:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788576005.526429:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576005.526431:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576005.526433:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788576005.526435:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576005.526437:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526439:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526447:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576005.526449:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.526452:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576005.526453:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576005.526455:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576005.526458:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526465:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576005.526467:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576005.526469:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576005.526471:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576005.526473:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576005.526476:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252096:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576005.526480:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576005.526482:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576005.526483:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576005.526485:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576005.526487:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788576005.526489:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576005.526493:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576005.526496:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576005.526499:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576005.526502:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393252096, portal 10
00000100:00000001:2.0:1788576005.526504:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576005.526506:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:2.0:1788576005.526512:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576030 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526519:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576005.526521:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576005.526523:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576005.526525:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393252096, offset 0
00000400:00000200:2.0:1788576005.526529:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576005.526533:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576005.526538:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576005.526541:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576005.526544:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576005.526546:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576005.526548:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576005.526551:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331225:331225:256:4294967295] 192.168.202.110@tcp LPNI seq info [331182:331182:8:4294967295]
00000400:00000200:2.0:1788576005.526556:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576005.526562:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576005.526566:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576005.526569:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576005.526573:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576005.526575:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788576005.526580:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576005.526582:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788576005.526586:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576005.526588:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576005.526591:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576005.526594:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576005.526597:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576005.526599:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576005.526602:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576005.526605:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576030 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526613:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576005.526616:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576030 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526622:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.526624:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576005.526627:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576005.526630:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576005.526633:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576005.526635:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307609127
00000100:00000001:2.0:1788576005.526638:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307610041
00000100:00000001:2.0:1788576005.526640:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307609127
00000100:00000001:2.0:1788576005.526642:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307610041
00000100:00000001:2.0:1788576005.526644:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576005.526646:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.526648:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576005.526653:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576005/real 1788576005]  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576005.526661:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576005.526663:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576005.526664:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576005.526667:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576005.526668:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576005.526670:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576005.526674:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576005.526695:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526702:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576005.526704:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526705:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576005.526707:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576005.526710:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526717:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576005.526721:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526727:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576005.526729:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576005.526733:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526734:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576005.526736:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576005.526739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576005.526742:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576005.526744:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576005.526748:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576005.526756:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252096:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576005.526760:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576005.526763:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526769:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576005.526771:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576005.526773:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576005.526775:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576005.526777:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576005.526779:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576005.526782:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576005.526784:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576005.526786:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576005.526788:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576005.526791:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788576005.526793:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576005.526797:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576005.526799:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576005.526802:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576005.526805:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393252224, portal 4
00000100:00000001:2.0:1788576005.526807:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576005.526808:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788576005.526814:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576030 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526820:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576005.526823:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576005.526825:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576005.526827:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393252224, offset 0
00000400:00000200:2.0:1788576005.526831:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576005.526835:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576005.526838:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576005.526841:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576005.526843:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576005.526846:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576005.526848:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576005.526851:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331226:331226:256:4294967295] 192.168.202.110@tcp LPNI seq info [331183:331183:8:4294967295]
00000400:00000200:2.0:1788576005.526854:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576005.526860:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576005.526865:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576005.526869:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576005.526873:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576005.526876:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788576005.526879:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576005.526882:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788576005.526886:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576005.526888:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576005.526890:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576005.526893:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576005.526896:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576005.526898:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576005.526900:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576005.526904:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576030 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526910:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576005.526913:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576030 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526918:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.526920:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576005.526923:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576005.526925:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576005.526928:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576005.526930:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307609273
00000100:00000001:2.0:1788576005.526932:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307610041
00000100:00000001:2.0:1788576005.526933:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307609273
00000100:00000001:2.0:1788576005.526935:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307610041
00000100:00000001:2.0:1788576005.526937:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576005.526938:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.526940:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576005.526943:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576005/real 1788576005]  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576005.526949:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576005.526950:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576005.526951:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576005.526954:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576005.526955:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576005.526957:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576005.526960:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576005.526967:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526973:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576005.526974:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.526976:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576005.526977:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576005.526981:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526987:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576005.526991:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.526996:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576005.526999:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.527000:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576005.527001:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.527003:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576005.527004:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576005.527007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576005.527010:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576005.527012:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576005.527015:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576005.527021:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576005.527025:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.527027:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576005.527029:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393252096/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.527035:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576005.527038:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576005.527040:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788576005.527042:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576005.527044:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576005.527045:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576005.527047:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576005.527049:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788576005.527052:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576005.527053:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576005.527054:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788576005.527057:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576005.527058:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.527059:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576005.527063:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393252224/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576030 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576005.527069:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576005.527070:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576005.527071:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788576005.527074:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576005.527075:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576005.527076:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576005.527078:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576005.527079:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788576005.527081:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576005.527082:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576005.527083:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:2.0:1788576005.527085:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576005.527086:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.527088:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576005.527091:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576005.527093:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576005.527095:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576005.527096:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576005.909517:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576005.909523:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576005.909527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576005.909528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576005.909530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576005.909531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576005.909792:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576005.909796:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576005.909800:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576006.037811:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576006.037819:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576006.037822:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576006.037827:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576006.037831:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576006.037835:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576006.037838:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576006.037840:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576006.037843:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576006.485505:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307610041, at 4307610048
00000100:00000040:0.0:1788576006.485506:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307610041, at 4307610048
00000100:00000010:1.0:1788576006.485520:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ccc0.
00000100:00000010:0.0:1788576006.485522:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c4e0.
00010000:00000001:3.0:1788576006.933503:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576006.933510:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576006.933515:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576006.933519:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576006.933521:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576006.933524:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576006.933526:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576006.933528:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576006.933530:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576006.933533:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576006.933536:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576006.933538:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576006.933539:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576006.933541:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576006.933543:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576006.933544:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576006.933545:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576006.933547:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576006.933599:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576006.933602:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576006.933604:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576007.061475:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576007.061482:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576007.061484:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576007.061490:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576007.061492:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576007.061495:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576007.061497:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576007.061500:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576007.061502:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576007.957533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576007.957541:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576007.957546:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576007.957547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576007.957552:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576007.957553:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576007.957669:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576007.957674:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576007.957677:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576008.085587:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576008.085595:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576008.085597:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576008.085603:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576008.085606:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576008.085610:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576008.085612:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576008.085614:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576008.085617:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576008.981530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576008.981539:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576008.981541:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576008.981545:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576008.981550:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576008.981552:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576008.981554:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576008.981556:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576008.981560:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576008.981562:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576008.981565:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576008.981567:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576008.981628:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576008.981632:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576008.981636:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576009.109512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576009.109520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576009.109522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576009.109527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576009.109531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576009.109535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576009.109538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576009.109541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576009.109543:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576010.006685:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576010.006692:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576010.006698:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576010.006699:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576010.006703:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576010.006704:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576010.006930:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576010.006934:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576010.006936:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576010.133674:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576010.133679:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576010.133681:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576010.133685:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576010.133688:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576010.133692:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.133694:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576010.133696:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.133698:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576010.645602:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576010.645622:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788576010.645625:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576010.645629:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576010.645633:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576010.645640:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12941
00000400:00000200:2.0:1788576010.645648:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576010.645653:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576010.645655:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576010.645658:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645660:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576010.645661:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645663:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576010.645665:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576010.645666:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788576010.645669:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576010.645671:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788576010.645672:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788576010.645673:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788576010.645675:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645676:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576010.645677:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645679:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576010.645681:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788576010.645684:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645685:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576010.645687:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576010.645689:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576010.645691:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576010.645693:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788576010.645694:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576010.645702:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01c00.
00000020:00000040:2.0:1788576010.645705:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788576010.645707:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576010.645708:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576010.645711:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576010.645712:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576010.645715:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576010.645721:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788576010.645726:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576010.645733:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576010.645740:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576010.645744:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576010.645747:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000de3acaa1] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576010.645763:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576010.645766:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576010.645769:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788576010.645771:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576010.645772:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576010.645775:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576010.645777:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12941
00000400:00000200:2.0:1788576010.645780:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576010.645783:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576010.645785:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1788576010.645788:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1788576010.645788:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576010.645791:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576010.645793:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576010.645799:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576010.645799:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576010.645801:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645803:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576010.645805:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576010.645806:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000100:00000001:0.0:1788576010.645808:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1788576010.645808:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576010.645810:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:0.0:1788576010.645811:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1788576010.645812:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:0.0:1788576010.645814:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:2.0:1788576010.645814:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
02000000:00000001:0.0:1788576010.645816:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1788576010.645817:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645818:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1788576010.645819:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576010.645820:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00100000:0.0:1788576010.645822:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393252352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576010.645823:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576010.645825:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788576010.645826:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1788576010.645827:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1788576010.645844:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576010.645846:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576010.645848:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576010.645853:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788576010.645855:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576010.645856:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576010.645857:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1788576010.645861:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788576010.645863:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:0.0:1788576010.645864:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
02000000:00000001:2.0:1788576010.645865:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576010.645866:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:0.0:1788576010.645871:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576010.645878:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393252352, portal 25
00000008:00100000:2.0:1788576010.645879:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1788576010.645881:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000008:00000001:2.0:1788576010.645881:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576010.645882:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000001:2.0:1788576010.645883:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576010.645886:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00e00.
00000100:00000040:0.0:1788576010.645888:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788576010.645889:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788576010.645891:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576010.645892:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576010.645894:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1788576010.645896:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1788576010.645896:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576010.645897:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576010.645899:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576010.645901:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
02000000:00000010:2.0:1788576010.645901:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000200:0.0:1788576010.645903:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393252352, offset 0
00000100:00000001:2.0:1788576010.645904:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576010.645909:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00100000:2.0:1788576010.645909:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576010.645916:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1788576010.645917:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576010.645918:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576010.645921:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5f665f0] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788576010.645923:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576010.645928:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788576010.645928:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576010.645931:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788576010.645932:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1788576010.645934:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788576010.645935:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1788576010.645936:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1788576010.645937:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1788576010.645939:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645940:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:0.0:1788576010.645942:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331227:331227:256:4294967295] 192.168.202.110@tcp LPNI seq info [331184:331184:8:4294967295]
00000100:00080000:2.0:1788576010.645942:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12941
00000400:00000200:2.0:1788576010.645945:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576010.645947:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576010.645948:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576010.645950:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576010.645953:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645954:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788576010.645955:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1788576010.645956:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645958:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576010.645960:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:0.0:1788576010.645961:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:2.0:1788576010.645962:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788576010.645963:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576010.645965:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000800:00000010:0.0:1788576010.645966:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000020:00000001:2.0:1788576010.645967:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788576010.645969:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000800:00000200:0.0:1788576010.645972:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788576010.645972:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645974:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1788576010.645976:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00000040:2.0:1788576010.645976:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645978:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576010.645980:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:0.1:1788576010.645981:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1788576010.645982:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576010.645984:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788576010.645984:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
02000000:00000001:2.0:1788576010.645986:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576010.645988:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000100:0.0:1788576010.645989:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788576010.645990:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788576010.645991:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000100:00000001:2.0:1788576010.645992:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576010.645994:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788576010.645995:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576010.645997:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1788576010.645997:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576010.646000:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03100.
00000400:00000200:0.0:1788576010.646002:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:2.0:1788576010.646002:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788576010.646005:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576010.646006:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576010.646007:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000100:00000001:2.0:1788576010.646007:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576010.646008:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1788576010.646009:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
02000000:00000001:2.0:1788576010.646010:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576010.646012:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000100:00000001:0.0:1788576010.646013:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000001:2.0:1788576010.646015:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576010.646017:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576010.646019:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576010.646026:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1788576010.646026:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576010.646028:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576010.646029:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788576010.646031:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008692924c] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646037:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576010.646038:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576010.646039:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000040:2.0:1788576010.646041:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1788576010.646043:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576010.646044:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:0.0:1788576010.646047:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788576010.646048:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12951)
00000100:00000001:0.0:1788576010.646049:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307615161
00000100:00000010:0.0:1788576010.646053:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576010.646059:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576010.646064:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307615161
00000100:00000010:0.0:1788576010.646067:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7240.
00000100:00000001:0.0:1788576010.646069:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576010.646072:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576010.646075:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576010.646077:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576010.646082:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576010/real 1788576010]  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576010.646096:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576010.646098:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576010.646100:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576010.646103:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576010.646104:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576010.646107:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576010.646110:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576010.646131:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646137:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576010.646139:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646141:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576010.646143:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576010.646147:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646154:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576010.646158:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646164:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576010.646166:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646167:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576010.646168:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646171:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576010.646173:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576010.646176:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576010.646179:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576010.646181:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576010.646185:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576010.646193:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393252352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576010.646198:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646200:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576010.646203:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393252352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646209:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576010.646211:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576010.646214:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788576010.646217:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576010.646219:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576010.646220:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576010.646222:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576010.646224:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788576010.646227:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576010.646229:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576010.646231:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:0.0:1788576010.646234:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576010.646235:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646237:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646244:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576010.646246:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576010.646248:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576010.646250:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576010.646251:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576010.646254:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646260:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576010.646261:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576010.646263:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576010.646265:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576010.646267:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576010.646269:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576010.646273:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576010.646275:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576010.646277:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576010.646278:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576010.646281:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788576010.646283:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576010.646286:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576010.646289:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576010.646292:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576010.646295:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393252480, portal 10
00000100:00000001:0.0:1788576010.646297:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576010.646298:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000040:0.0:1788576010.646305:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646311:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576010.646313:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576010.646315:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576010.646318:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393252480, offset 0
00000400:00000200:0.0:1788576010.646322:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576010.646474:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576010.646480:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576010.646483:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576010.646486:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576010.646488:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576010.646490:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576010.646493:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331228:331228:256:4294967295] 192.168.202.110@tcp LPNI seq info [331185:331185:8:4294967295]
00000400:00000200:0.0:1788576010.646497:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576010.646503:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576010.646507:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576010.646511:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576010.646515:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576010.646519:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1788576010.646523:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576010.646525:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1788576010.646528:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576010.646530:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576010.646533:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576010.646536:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576010.646540:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576010.646541:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576010.646544:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576010.646548:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646617:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576010.646621:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646627:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576010.646629:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576010.646633:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576010.646636:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576010.646640:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576010.646642:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307614247
00000100:00000001:0.0:1788576010.646645:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307615161
00000100:00000001:0.0:1788576010.646647:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307614247
00000100:00000001:0.0:1788576010.646649:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307615161
00000100:00000001:0.0:1788576010.646651:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576010.646653:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576010.646655:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576010.646659:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576010/real 1788576010]  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576010.646666:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576010.646668:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576010.646670:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576010.646673:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576010.646674:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576010.646677:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576010.646680:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576010.646688:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646694:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576010.646695:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646697:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576010.646698:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576010.646701:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646707:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576010.646711:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646717:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576010.646720:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646721:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576010.646722:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646724:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576010.646726:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576010.646728:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576010.646731:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576010.646733:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576010.646736:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576010.646744:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576010.646748:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576010.646751:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646757:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576010.646758:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576010.646760:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576010.646762:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576010.646764:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576010.646767:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252608:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576010.646769:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576010.646771:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576010.646773:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576010.646774:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576010.646779:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788576010.646781:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576010.646784:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576010.646787:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576010.646790:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576010.646793:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393252608, portal 4
00000100:00000001:0.0:1788576010.646794:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576010.646795:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:0.0:1788576010.646801:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646807:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576010.646809:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576010.646811:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576010.646813:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393252608, offset 0
00000400:00000200:0.0:1788576010.646817:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576010.646821:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576010.646824:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576010.646827:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576010.646841:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576010.646844:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576010.646846:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576010.646849:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331229:331229:256:4294967295] 192.168.202.110@tcp LPNI seq info [331186:331186:8:4294967295]
00000400:00000200:0.0:1788576010.646852:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576010.646858:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576010.646863:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576010.646866:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576010.646870:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576010.646872:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1788576010.646876:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576010.646879:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1788576010.646882:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576010.646884:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576010.646887:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576010.646890:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576010.646893:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576010.646894:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576010.646897:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576010.646900:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646908:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576010.646911:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646917:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576010.646919:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576010.646922:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576010.646924:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576010.646928:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576010.646929:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307614393
00000100:00000001:0.0:1788576010.646931:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307615161
00000100:00000001:0.0:1788576010.646933:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307614393
00000100:00000001:0.0:1788576010.646936:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307615161
00000100:00000001:0.0:1788576010.646937:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576010.646939:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576010.646941:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576010.646945:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576010/real 1788576010]  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576010.646952:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576010.646953:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576010.646955:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576010.646957:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576010.646958:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576010.646960:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576010.646963:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576010.646970:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646975:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576010.646977:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.646978:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576010.646980:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576010.646983:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646989:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576010.646993:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.646999:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576010.647001:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.647003:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576010.647004:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.647005:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576010.647007:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576010.647010:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576010.647013:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576010.647015:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576010.647018:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576010.647025:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252608:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576010.647029:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.647031:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576010.647034:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393252480/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.647039:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576010.647040:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576010.647042:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788576010.647044:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576010.647045:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576010.647045:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576010.647047:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576010.647048:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788576010.647049:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576010.647050:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576010.647051:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:0.0:1788576010.647053:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576010.647053:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.647054:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576010.647056:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393252608/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576010.647060:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576010.647061:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576010.647062:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788576010.647063:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576010.647064:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576010.647065:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576010.647066:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576010.647067:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788576010.647068:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576010.647068:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576010.647069:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:0.0:1788576010.647070:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576010.647071:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.647072:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576010.647075:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576010.647076:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576010.647077:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576010.647078:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576011.029514:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576011.029520:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576011.029525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576011.029526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576011.029529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576011.029531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576011.029594:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576011.029598:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576011.029601:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576011.157515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576011.157523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576011.157525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576011.157530:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576011.157534:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576011.157537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576011.157540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576011.157542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576011.157546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576011.605514:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307615161, at 4307615168
00000100:00000010:0.0:1788576011.605536:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000040:1.0:1788576011.605833:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307615161, at 4307615168
00000100:00000010:1.0:1788576011.605839:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7240.
00010000:00000001:3.0:1788576012.053535:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576012.053542:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576012.053548:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576012.053550:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576012.053553:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576012.053554:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576012.053750:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576012.053753:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576012.053757:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576012.181477:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576012.181485:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576012.181487:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576012.181492:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576012.181496:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576012.181499:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576012.181501:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576012.181504:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576012.181507:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576013.077651:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576013.077661:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576013.077667:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576013.077668:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576013.077673:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576013.077674:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576013.077792:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576013.077797:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576013.077800:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576013.205668:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576013.205676:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576013.205678:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576013.205683:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576013.205686:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576013.205691:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576013.205694:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576013.205697:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576013.205701:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576014.101544:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576014.101553:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576014.101561:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576014.101562:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576014.101566:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576014.101567:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576014.101636:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576014.101640:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576014.101643:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576014.229574:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576014.229584:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576014.229587:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576014.229595:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576014.229599:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576014.229604:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576014.229607:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576014.229610:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576014.229614:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576015.125545:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576015.125552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576015.125556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576015.125557:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576015.125560:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576015.125561:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576015.125660:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576015.125663:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576015.125666:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576015.253546:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576015.253555:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576015.253557:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576015.253563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576015.253566:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576015.253571:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.253574:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576015.253576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.253579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576015.765476:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576015.765491:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788576015.765494:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576015.765499:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576015.765505:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576015.765514:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12946
00000400:00000200:2.0:1788576015.765525:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576015.765532:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576015.765536:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576015.765539:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.765543:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576015.765545:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.765549:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576015.765554:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576015.765557:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788576015.765560:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576015.765564:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788576015.765567:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788576015.765569:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788576015.765572:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.765574:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576015.765576:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.765578:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576015.765581:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788576015.765585:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576015.765588:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576015.765592:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576015.765596:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576015.765599:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576015.765602:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788576015.765604:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576015.765615:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02d80.
00000020:00000040:2.0:1788576015.765619:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788576015.765624:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576015.765626:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576015.765629:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576015.765632:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576015.765636:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576015.765644:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11400.
00000100:00000001:2.0:1788576015.765654:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576015.765665:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576015.765676:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576015.765682:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576015.765695:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009a9ccdb8] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576015.765726:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576015.765729:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576015.765733:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788576015.765735:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576015.765736:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576015.765739:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576015.765742:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12946
00000400:00000200:2.0:1788576015.765746:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576015.765750:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576015.765753:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576015.765756:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.765759:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576015.765761:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.765764:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576015.765766:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1788576015.765768:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1788576015.765768:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788576015.765771:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788576015.765775:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1788576015.765775:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788576015.765779:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788576015.765780:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788576015.765782:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788576015.765788:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000040:0.0:1788576015.765791:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576015.765800:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576015.765804:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576015.765806:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576015.765808:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788576015.765810:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788576015.765812:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:0.0:1788576015.765816:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:2.0:1788576015.765817:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1788576015.765821:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576015.765822:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576015.765826:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576015.765827:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1788576015.765834:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393252736:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1788576015.765836:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576015.765841:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.765842:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1788576015.765844:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1788576015.765848:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:2.0:1788576015.765850:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788576015.765853:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1788576015.765854:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576015.765857:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1788576015.765858:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576015.765863:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03b80.
02000000:00000010:0.0:1788576015.765865:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
00000020:00000040:2.0:1788576015.765867:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788576015.765871:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576015.765872:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576015.765875:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576015.765877:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576015.765879:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1788576015.765882:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788576015.765882:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576015.765888:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
02000000:00000010:2.0:1788576015.765890:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12800.
00000100:00000001:2.0:1788576015.765895:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576015.765899:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00100000:2.0:1788576015.765902:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576015.765911:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576015.765913:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576015.765914:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393252736, portal 25
00000100:00000040:2.0:1788576015.765918:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037d167ca] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.765919:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576015.765928:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.765931:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00080000:2.0:1788576015.765933:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576015.765938:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788576015.765941:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:0.0:1788576015.765943:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1788576015.765944:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576015.765950:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576015.765956:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12946
00000100:00000001:0.0:1788576015.765959:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:2.0:1788576015.765963:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788576015.765966:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576015.765970:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:0.0:1788576015.765971:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00080000:2.0:1788576015.765973:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1788576015.765977:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393252736, offset 0
00000100:00000040:2.0:1788576015.765978:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.765982:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576015.765996:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.765999:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576015.766001:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576015.766003:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:0.0:1788576015.766006:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:2.0:1788576015.766007:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576015.766011:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788576015.766013:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788576015.766015:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788576015.766019:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788576015.766020:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.766025:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576015.766028:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788576015.766031:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1788576015.766032:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576015.766037:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:0.0:1788576015.766040:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1788576015.766042:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576015.766046:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576015.766049:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:2.0:1788576015.766050:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576015.766055:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1788576015.766056:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:2.0:1788576015.766059:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576015.766061:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1788576015.766063:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576015.766067:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:0.0:1788576015.766069:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331230:331230:256:4294967295] 192.168.202.110@tcp LPNI seq info [331187:331187:8:4294967295]
00000008:00000001:2.0:1788576015.766071:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576015.766075:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576015.766080:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1788576015.766081:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00000.
00000020:00000040:2.0:1788576015.766085:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788576015.766087:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576015.766089:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576015.766091:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576015.766092:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576015.766094:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576015.766099:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000400:00000200:0.0:1788576015.766104:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1788576015.766105:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576015.766113:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1788576015.766117:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788576015.766126:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000010:0.0:1788576015.766127:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000100:00000001:2.0:1788576015.766129:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576015.766135:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bf5936f] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1788576015.766139:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576015.766146:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1788576015.766148:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576015.766154:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1788576015.766156:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1788576015.766161:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:0.1:1788576015.766162:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788576015.766169:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788576015.766170:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12957)
00000800:00000010:0.0:1788576015.766172:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576015.766181:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576015.766187:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576015.766193:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576015.766196:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576015.766202:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576015.766206:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766216:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576015.766220:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766227:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.766229:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576015.766235:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576015.766240:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576015.766243:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307620281
00000100:00000010:0.0:1788576015.766249:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576015.766256:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576015.766263:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307620281
00000100:00000010:0.0:1788576015.766266:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7f60.
00000100:00000001:0.0:1788576015.766269:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576015.766273:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576015.766276:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.766279:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576015.766284:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576015/real 1788576015]  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576015.766300:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576015.766302:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576015.766304:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576015.766307:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576015.766308:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576015.766311:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576015.766315:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576015.766323:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766330:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576015.766331:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.766334:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576015.766337:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576015.766341:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766349:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576015.766354:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766361:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576015.766364:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.766366:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576015.766367:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.766369:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576015.766371:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576015.766374:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576015.766378:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576015.766381:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576015.766385:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576015.766417:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393252736:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576015.766422:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.766424:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576015.766427:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393252736/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766434:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576015.766436:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576015.766439:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788576015.766443:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576015.766445:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576015.766446:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576015.766449:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576015.766451:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788576015.766455:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576015.766457:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576015.766459:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:0.0:1788576015.766463:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576015.766464:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.766467:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.766475:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576015.766477:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.766480:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576015.766481:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576015.766482:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576015.766486:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766492:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576015.766493:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576015.766496:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576015.766497:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576015.766500:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576015.766502:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252864:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576015.766506:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576015.766508:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576015.766510:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576015.766511:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576015.766514:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788576015.766516:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576015.766529:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576015.766532:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576015.766536:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576015.766539:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393252864, portal 10
00000100:00000001:0.0:1788576015.766541:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576015.766542:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000040:0.0:1788576015.766549:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766555:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576015.766558:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576015.766560:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576015.766563:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393252864, offset 0
00000400:00000200:0.0:1788576015.766567:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576015.766570:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576015.766574:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576015.766577:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576015.766580:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576015.766582:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576015.766584:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576015.766588:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331231:331231:256:4294967295] 192.168.202.110@tcp LPNI seq info [331188:331188:8:4294967295]
00000400:00000200:0.0:1788576015.766592:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576015.766598:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576015.766603:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576015.766606:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576015.766609:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576015.766613:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788576015.766617:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576015.766619:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788576015.766623:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576015.766624:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576015.766627:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576015.766630:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576015.766634:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576015.766635:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576015.766638:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576015.766641:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766649:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576015.766652:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766658:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.766660:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576015.766663:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576015.766666:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576015.766670:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576015.766672:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307619367
00000100:00000001:0.0:1788576015.766675:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307620281
00000100:00000001:0.0:1788576015.766677:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307619367
00000100:00000001:0.0:1788576015.766679:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307620281
00000100:00000001:0.0:1788576015.766681:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576015.766683:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.766684:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576015.766688:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576015/real 1788576015]  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576015.766696:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576015.766698:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576015.766700:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576015.766703:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576015.766704:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576015.766706:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576015.766710:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576015.766718:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766724:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576015.766726:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.766728:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576015.766729:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576015.766733:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766739:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576015.766743:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766750:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576015.766753:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.766754:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576015.766756:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.766757:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576015.766759:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576015.766762:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576015.766766:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576015.766768:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576015.766771:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576015.766780:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252864:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576015.766783:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576015.766787:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766793:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576015.766795:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576015.766797:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576015.766799:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576015.766801:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576015.766804:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252992:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576015.766807:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576015.766809:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576015.766810:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576015.766812:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576015.766815:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788576015.766817:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576015.766821:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576015.766824:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576015.766827:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576015.766829:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393252992, portal 4
00000100:00000001:0.0:1788576015.766831:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576015.766832:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:0.0:1788576015.766838:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766844:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576015.766847:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576015.766849:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576015.766851:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393252992, offset 0
00000400:00000200:0.0:1788576015.766855:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576015.766859:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576015.766862:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576015.766865:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576015.766868:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576015.766870:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576015.766872:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576015.766875:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331232:331232:256:4294967295] 192.168.202.110@tcp LPNI seq info [331189:331189:8:4294967295]
00000400:00000200:0.0:1788576015.766880:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576015.766885:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576015.766889:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576015.766892:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576015.766896:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576015.766898:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1788576015.766902:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576015.766905:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1788576015.766909:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576015.766911:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576015.766914:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576015.766917:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576015.766920:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576015.766922:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576015.766924:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576015.766928:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766936:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576015.766939:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.766944:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.766946:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576015.766949:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576015.766952:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576015.766955:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576015.766957:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307619513
00000100:00000001:0.0:1788576015.766960:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307620281
00000100:00000001:0.0:1788576015.766962:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307619513
00000100:00000001:0.0:1788576015.766964:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307620281
00000100:00000001:0.0:1788576015.766966:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576015.766968:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.766970:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576015.766974:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576015/real 1788576015]  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576015.766981:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576015.766983:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576015.766995:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576015.766998:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576015.767000:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576015.767002:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576015.767006:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576015.767013:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.767020:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576015.767021:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.767023:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576015.767025:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576015.767028:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.767035:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576015.767039:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.767046:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576015.767048:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.767050:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576015.767051:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.767053:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576015.767055:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576015.767058:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576015.767061:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576015.767063:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576015.767068:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576015.767075:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393252992:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576015.767079:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.767081:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576015.767084:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393252864/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.767090:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576015.767092:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576015.767093:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788576015.767096:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576015.767098:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576015.767099:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576015.767101:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576015.767103:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12800.
02000000:00000001:0.0:1788576015.767105:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576015.767106:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576015.767108:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:0.0:1788576015.767110:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576015.767111:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.767113:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576015.767116:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393252992/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576015.767122:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576015.767124:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576015.767125:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788576015.767128:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576015.767129:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576015.767130:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576015.767132:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576015.767133:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788576015.767135:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576015.767136:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576015.767138:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:0.0:1788576015.767140:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576015.767141:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.767142:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576015.767147:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576015.767148:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576015.767150:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576015.767151:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576016.150114:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576016.150121:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576016.150126:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576016.150127:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576016.150130:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576016.150132:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576016.150257:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576016.150262:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576016.150266:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576016.277495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576016.277502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576016.277504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576016.277510:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576016.277513:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576016.277516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576016.277519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576016.277521:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576016.277523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576016.725529:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307620281, at 4307620288
00000100:00000040:0.0:1788576016.725532:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307620281, at 4307620288
00000100:00000010:0.0:1788576016.725548:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:1.0:1788576016.725551:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f60.
00010000:00000001:3.0:1788576017.173520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576017.173527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576017.173531:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576017.173534:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576017.173536:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576017.173538:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576017.173541:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576017.173542:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576017.173544:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576017.173545:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576017.173548:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576017.173549:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576017.173550:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576017.173551:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576017.173552:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576017.173553:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576017.173554:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576017.173556:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576017.173612:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576017.173616:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576017.173618:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576017.301521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576017.301528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576017.301529:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576017.301535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576017.301539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576017.301542:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576017.301545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576017.301547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576017.301550:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788576017.813576:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788576017.813587:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788576017.813593:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788576017.813783:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1788576017.813801:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788576017.813803:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788576017.813810:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788576017.813815:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788576018.197540:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576018.197547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576018.197552:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576018.197553:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576018.197556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576018.197557:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576018.197632:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576018.197637:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576018.197641:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576018.325722:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576018.325734:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576018.325737:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576018.325745:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576018.325750:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576018.325755:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576018.325759:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576018.325762:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576018.325766:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576019.224240:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576019.224251:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576019.224254:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576019.224258:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576019.224263:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576019.224266:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576019.224268:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576019.224271:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576019.224275:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576019.224277:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576019.224280:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576019.224282:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576019.224622:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576019.224627:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576019.224630:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576019.349564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576019.349575:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576019.349577:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576019.349583:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576019.349587:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576019.349592:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576019.349595:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576019.349598:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576019.349601:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576020.245849:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576020.245859:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576020.245866:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576020.245867:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576020.245871:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576020.245872:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576020.246006:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576020.246011:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576020.246015:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576020.373550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576020.373560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576020.373562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576020.373569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576020.373573:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576020.373578:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.373581:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576020.373584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.373588:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576020.885544:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576020.885557:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576020.885561:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576020.885566:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576020.885572:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576020.885580:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12952
00000400:00000200:3.0:1788576020.885592:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576020.885600:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576020.885609:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576020.885613:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576020.885616:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576020.885619:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576020.885621:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576020.885626:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576020.885628:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576020.885632:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576020.885635:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576020.885638:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576020.885641:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576020.885644:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576020.885646:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576020.885648:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576020.885650:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576020.885654:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576020.885658:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576020.885661:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576020.885665:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576020.885669:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576020.885672:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576020.885675:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576020.885678:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576020.885691:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788576020.885695:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576020.885700:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576020.885702:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576020.885706:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576020.885708:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576020.885713:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576020.885723:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788576020.885733:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576020.885745:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576020.885756:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576020.885762:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576020.885767:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576020.885791:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576020.885794:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576020.885798:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576020.885800:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576020.885801:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576020.885805:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576020.885808:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12952
00000400:00000200:3.0:1788576020.885812:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576020.885816:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576020.885819:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576020.885822:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576020.885825:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788576020.885825:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576020.885830:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576020.885833:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788576020.885834:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788576020.885837:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788576020.885838:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576020.885842:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576020.885844:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:0.0:1788576020.885849:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1788576020.885857:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:0.0:1788576020.885860:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:3.0:1788576020.885862:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576020.885866:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788576020.885869:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576020.885872:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576020.885872:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576020.885877:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:0.0:1788576020.885880:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788576020.885880:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:0.0:1788576020.885884:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1788576020.885885:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:0.0:1788576020.885889:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576020.885891:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:0.0:1788576020.885896:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393253120:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788576020.885897:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576020.885902:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788576020.885905:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576020.885911:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576020.885911:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576020.885915:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576020.885916:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576020.885920:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576020.885925:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788576020.885927:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:3.0:1788576020.885931:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576020.885933:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576020.885936:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
00000100:00000010:3.0:1788576020.885937:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
02000000:00000001:0.0:1788576020.885939:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788576020.885942:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788576020.885946:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576020.885948:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576020.885950:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576020.885951:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576020.885954:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1788576020.885956:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
02000000:00000001:3.0:1788576020.885957:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576020.885965:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000400:00000010:0.0:1788576020.885968:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000001:3.0:1788576020.885970:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576020.885978:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393253120, portal 25
00000100:00100000:3.0:1788576020.885979:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576020.885987:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576020.885991:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576020.885992:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000001:3.0:1788576020.885996:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576020.886002:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788576020.886004:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886021:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1788576020.886026:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576020.886027:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:3.0:1788576020.886031:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1788576020.886032:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576020.886037:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393253120, offset 0
00000100:00080000:3.0:1788576020.886038:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788576020.886042:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576020.886044:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1788576020.886047:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576020.886049:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576020.886054:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12952
00000400:00000200:3.0:1788576020.886060:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576020.886063:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576020.886065:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1788576020.886069:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788576020.886070:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576020.886075:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576020.886077:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788576020.886080:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788576020.886081:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576020.886086:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576020.886088:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576020.886090:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1788576020.886092:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:3.0:1788576020.886093:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788576020.886097:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788576020.886099:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:3.0:1788576020.886102:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788576020.886106:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1788576020.886110:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788576020.886113:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788576020.886115:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576020.886121:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788576020.886122:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331233:331233:256:4294967295] 192.168.202.110@tcp LPNI seq info [331190:331190:8:4294967295]
00000100:00000001:3.0:1788576020.886126:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788576020.886130:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1788576020.886130:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788576020.886136:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576020.886152:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576020.886155:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576020.886159:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1788576020.886163:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1788576020.886164:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576020.886169:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576020.886172:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:0.0:1788576020.886175:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00000001:3.0:1788576020.886176:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576020.886180:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788576020.886183:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000100:00000010:3.0:1788576020.886185:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788576020.886189:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788576020.886192:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576020.886195:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788576020.886196:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788576020.886198:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576020.886201:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:0.0:1788576020.886204:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:3.0:1788576020.886204:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576020.886210:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000800:00000200:0.1:1788576020.886215:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788576020.886217:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788576020.886221:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00100000:3.0:1788576020.886226:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:0.0:1788576020.886230:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576020.886236:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000100:00000001:3.0:1788576020.886239:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576020.886243:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576020.886247:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1788576020.886249:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788576020.886256:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1788576020.886263:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576020.886268:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000100:00000040:3.0:1788576020.886268:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1788576020.886273:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000040:3.0:1788576020.886275:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788576020.886282:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12962)
00000100:00000001:0.0:1788576020.886298:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576020.886304:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886315:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576020.886319:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886327:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576020.886329:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576020.886336:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576020.886340:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576020.886343:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307625401
00000100:00000010:0.0:1788576020.886349:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576020.886356:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576020.886364:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307625401
00000100:00000010:0.0:1788576020.886366:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7060.
00000100:00000001:0.0:1788576020.886370:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576020.886374:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576020.886377:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576020.886379:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576020.886383:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576020/real 1788576020]  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576020.886414:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576020.886416:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576020.886419:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576020.886422:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576020.886424:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576020.886426:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576020.886430:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576020.886438:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886445:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576020.886447:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.886450:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576020.886451:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576020.886455:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886463:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576020.886474:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886484:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576020.886487:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.886489:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576020.886491:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.886493:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576020.886495:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576020.886498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576020.886502:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576020.886504:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576020.886509:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576020.886516:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393253120:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576020.886521:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.886523:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576020.886526:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393253120/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886532:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576020.886535:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576020.886538:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788576020.886542:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576020.886544:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576020.886545:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576020.886548:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576020.886551:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:0.0:1788576020.886554:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576020.886556:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576020.886558:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:0.0:1788576020.886562:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576020.886563:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.886566:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.886574:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576020.886576:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576020.886578:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576020.886580:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576020.886581:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576020.886585:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886592:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576020.886594:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576020.886596:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576020.886598:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576020.886600:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576020.886603:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393253248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576020.886606:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576020.886608:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576020.886610:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576020.886612:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576020.886614:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788576020.886616:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576020.886620:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576020.886623:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576020.886627:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576020.886630:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393253248, portal 10
00000100:00000001:0.0:1788576020.886631:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576020.886633:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000040:0.0:1788576020.886639:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886646:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576020.886648:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576020.886650:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576020.886653:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393253248, offset 0
00000400:00000200:0.0:1788576020.886657:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576020.886661:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576020.886665:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576020.886668:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576020.886671:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576020.886673:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576020.886676:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576020.886679:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331234:331234:256:4294967295] 192.168.202.110@tcp LPNI seq info [331191:331191:8:4294967295]
00000400:00000200:0.0:1788576020.886683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576020.886690:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576020.886694:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576020.886697:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576020.886701:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576020.886704:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788576020.886708:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576020.886711:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788576020.886729:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576020.886731:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576020.886735:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576020.886738:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576020.886741:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576020.886743:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576020.886746:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576020.886749:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886757:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576020.886760:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886766:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576020.886767:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576020.886770:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576020.886775:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576020.886778:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576020.886780:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307624487
00000100:00000001:0.0:1788576020.886784:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307625401
00000100:00000001:0.0:1788576020.886786:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307624487
00000100:00000001:0.0:1788576020.886788:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307625401
00000100:00000001:0.0:1788576020.886790:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576020.886793:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576020.886794:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576020.886798:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576020/real 1788576020]  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576020.886806:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576020.886807:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576020.886809:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576020.886811:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576020.886813:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576020.886815:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576020.886818:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576020.886828:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886835:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576020.886836:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.886838:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576020.886840:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576020.886844:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886850:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576020.886855:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886861:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576020.886864:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.886865:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576020.886867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.886869:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576020.886871:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576020.886873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576020.886877:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576020.886879:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576020.886883:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576020.886891:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393253248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576020.886895:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576020.886898:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886905:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576020.886906:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576020.886909:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576020.886910:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576020.886912:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576020.886915:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393253376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576020.886918:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576020.886920:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576020.886921:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576020.886923:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576020.886926:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788576020.886929:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576020.886933:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576020.886936:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576020.886940:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576020.886943:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393253376, portal 4
00000100:00000001:0.0:1788576020.886945:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576020.886946:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:0.0:1788576020.886954:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.886960:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576020.886963:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576020.886965:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576020.886967:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393253376, offset 0
00000400:00000200:0.0:1788576020.886971:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576020.886975:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576020.886978:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576020.886981:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576020.886984:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576020.886986:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576020.886988:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576020.886991:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331235:331235:256:4294967295] 192.168.202.110@tcp LPNI seq info [331192:331192:8:4294967295]
00000400:00000200:0.0:1788576020.886995:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576020.887001:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576020.887006:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576020.887009:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576020.887012:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576020.887015:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1788576020.887019:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576020.887022:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1788576020.887026:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576020.887028:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576020.887031:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576020.887034:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576020.887038:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576020.887039:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576020.887042:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576020.887045:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.887053:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576020.887056:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.887062:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576020.887064:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576020.887067:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576020.887071:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576020.887073:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576020.887075:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307624633
00000100:00000001:0.0:1788576020.887078:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307625401
00000100:00000001:0.0:1788576020.887080:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307624633
00000100:00000001:0.0:1788576020.887082:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307625401
00000100:00000001:0.0:1788576020.887084:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576020.887087:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576020.887088:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576020.887092:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576020/real 1788576020]  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576020.887100:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576020.887101:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576020.887103:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576020.887105:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576020.887106:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576020.887108:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576020.887111:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576020.887119:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.887125:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576020.887127:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.887129:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576020.887131:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576020.887134:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.887154:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576020.887157:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.887165:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576020.887167:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.887169:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576020.887170:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.887172:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576020.887173:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576020.887176:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576020.887179:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576020.887181:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576020.887185:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576020.887193:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393253376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576020.887196:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.887198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576020.887201:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393253248/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.887207:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576020.887209:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576020.887211:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788576020.887214:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576020.887215:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576020.887217:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576020.887219:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576020.887220:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:0.0:1788576020.887223:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576020.887224:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576020.887225:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:0.0:1788576020.887228:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576020.887230:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.887232:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576020.887235:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393253376/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576020.887241:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576020.887242:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576020.887243:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788576020.887245:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576020.887246:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576020.887248:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576020.887250:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576020.887251:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788576020.887253:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576020.887254:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576020.887255:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:0.0:1788576020.887258:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576020.887259:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.887261:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576020.887264:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576020.887265:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576020.887267:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576020.887269:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576021.269558:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576021.269568:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576021.269574:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576021.269575:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576021.269579:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576021.269581:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576021.269814:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576021.269818:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576021.269821:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576021.397563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576021.397574:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576021.397577:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576021.397584:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576021.397589:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576021.397594:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576021.397597:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576021.397600:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576021.397604:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576021.845527:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307625401, at 4307625408
00000100:00000040:1.0:1788576021.845527:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307625401, at 4307625408
00000100:00000010:0.0:1788576021.845550:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:1.0:1788576021.845552:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7060.
00010000:00000001:3.0:1788576022.293516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576022.293522:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576022.293527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576022.293533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576022.293537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576022.293538:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576022.293627:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576022.293634:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576022.293638:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576022.425016:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576022.425024:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576022.425026:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576022.425032:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576022.425036:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576022.425040:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576022.425043:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576022.425045:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576022.425048:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576023.317535:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576023.317542:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576023.317548:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576023.317549:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576023.317553:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576023.317554:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576023.317668:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576023.317672:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576023.317674:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576023.445537:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576023.445544:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576023.445546:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576023.445551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576023.445556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576023.445560:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576023.445562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576023.445566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576023.445569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576024.341481:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576024.341488:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576024.341494:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576024.341495:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576024.341497:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576024.341498:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576024.341788:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576024.341790:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576024.341793:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576024.469666:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576024.469671:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576024.469674:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576024.469679:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576024.469683:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576024.469686:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576024.469689:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576024.469691:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576024.469695:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576025.365522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576025.365527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576025.365530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576025.365531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576025.365533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576025.365534:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576025.365594:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576025.365597:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576025.365601:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576025.493588:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576025.493595:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576025.493597:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576025.493602:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576025.493606:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576025.493610:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576025.493612:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576025.493614:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576025.493617:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576026.005553:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576026.005563:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788576026.005574:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576026.005579:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576026.005583:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576026.005590:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12957
00000400:00000200:1.0:1788576026.005600:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576026.005606:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576026.005609:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576026.005612:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.005614:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576026.005617:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.005619:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576026.005623:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576026.005625:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788576026.005627:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576026.005630:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788576026.005633:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788576026.005634:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788576026.005637:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.005639:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576026.005641:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.005643:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576026.005646:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788576026.005650:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576026.005652:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576026.005655:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576026.005658:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576026.005661:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576026.005663:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788576026.005666:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576026.005676:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788576026.005679:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788576026.005683:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576026.005685:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576026.005687:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576026.005690:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576026.005694:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576026.005702:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788576026.005709:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576026.005719:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576026.005728:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576026.005733:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576026.005738:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576026.005761:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576026.005765:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576026.005768:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788576026.005770:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576026.005771:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576026.005774:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576026.005777:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12957
00000400:00000200:1.0:1788576026.005781:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576026.005784:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576026.005786:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576026.005788:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.005790:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576026.005792:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.005794:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576026.005797:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576026.005798:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788576026.005799:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576026.005801:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788576026.005803:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788576026.005806:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788576026.005809:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.005811:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576026.005813:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.005815:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576026.005818:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788576026.005819:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1788576026.005821:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576026.005825:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576026.005826:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576026.005829:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1788576026.005830:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576026.005837:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1788576026.005838:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788576026.005840:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576026.005845:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576026.005850:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1788576026.005852:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576026.005856:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1788576026.005857:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576026.005861:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788576026.005861:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576026.005865:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:1.0:1788576026.005865:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
02000000:00000001:0.0:1788576026.005881:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788576026.005882:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788576026.005888:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:0.0:1788576026.005889:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393253504:192.168.202.110@tcp:250:kworker.0
02000000:00000001:1.0:1788576026.005890:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576026.005895:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1788576026.005897:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1788576026.005898:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576026.005902:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576026.005903:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:1.0:1788576026.005906:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
02000000:00000001:0.0:1788576026.005907:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576026.005912:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788576026.005913:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576026.005918:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576026.005922:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576026.005923:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788576026.005930:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576026.005936:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000100:00000001:1.0:1788576026.005937:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576026.005941:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576026.005944:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000040:1.0:1788576026.005947:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576026.005951:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393253504, portal 25
00000100:00000001:0.0:1788576026.005956:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576026.005959:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
00000100:00000001:1.0:1788576026.005961:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576026.005965:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788576026.005968:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1788576026.005969:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576026.005976:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576026.005979:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576026.005981:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000001:1.0:1788576026.005985:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:0.0:1788576026.005986:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393253504, offset 0
00000100:00080000:1.0:1788576026.005988:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576026.005993:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:0.0:1788576026.005995:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:1.0:1788576026.005999:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12957
00000400:00000200:0.0:1788576026.006005:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576026.006011:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576026.006024:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576026.006027:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576026.006031:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576026.006034:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576026.006039:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331236:331236:256:4294967295] 192.168.202.110@tcp LPNI seq info [331193:331193:8:4294967295]
00000400:00000200:0.0:1788576026.006045:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576026.006056:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576026.006062:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576026.006067:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576026.006074:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576026.006078:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788576026.006093:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576026.006096:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788576026.006101:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576026.006103:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576026.006109:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576026.006114:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1788576026.006255:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576026.006260:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576026.006263:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576026.006265:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.006267:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576026.006269:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.006271:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576026.006273:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576026.006275:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788576026.006276:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576026.006278:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788576026.006280:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788576026.006282:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788576026.006285:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.006287:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576026.006289:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576026.006303:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576026.006306:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788576026.006309:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576026.006311:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576026.006313:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576026.006317:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576026.006319:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576026.006321:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576026.006323:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788576026.006326:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576026.006328:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576026.006331:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788576026.006333:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788576026.006335:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576026.006337:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576026.006350:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576026.006351:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576026.006353:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576026.006358:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788576026.006361:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576026.006366:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576026.006373:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576026.006374:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576026.006378:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576026.006385:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576026.006584:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788576026.006588:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788576026.006593:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12967)
00000400:00000200:0.0:1788576026.007086:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576026.007090:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576026.007094:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576026.007098:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007106:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576026.007110:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007118:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576026.007120:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576026.007126:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576026.007130:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576026.007133:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307630522
00000100:00000010:0.0:1788576026.007139:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576026.007145:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576026.007150:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307630522
00000100:00000010:0.0:1788576026.007153:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7540.
00000100:00000001:0.0:1788576026.007156:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576026.007159:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576026.007162:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576026.007165:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576026.007169:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576026/real 1788576026]  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576026.007178:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576026.007179:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576026.007181:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576026.007184:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576026.007185:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576026.007187:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576026.007190:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576026.007198:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007203:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576026.007205:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007208:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576026.007209:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576026.007213:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007219:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576026.007224:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007230:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576026.007232:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007234:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576026.007235:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007237:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576026.007239:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576026.007241:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576026.007245:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576026.007248:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576026.007252:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576026.007260:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393253504:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576026.007263:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007265:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576026.007267:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393253504/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007271:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576026.007273:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576026.007275:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576026.007277:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576026.007278:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576026.007279:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576026.007281:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576026.007283:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:0.0:1788576026.007286:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576026.007287:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576026.007288:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:0.0:1788576026.007290:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576026.007290:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007292:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007439:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576026.007441:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576026.007442:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576026.007443:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576026.007444:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576026.007448:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007453:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576026.007454:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576026.007456:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576026.007457:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576026.007459:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576026.007461:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393253632:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576026.007463:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576026.007464:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576026.007466:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576026.007467:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576026.007469:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576026.007470:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576026.007473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576026.007475:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576026.007478:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576026.007480:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393253632, portal 10
00000100:00000001:0.0:1788576026.007481:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576026.007482:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345473920 : -130483364077696 : ffff8953796f5f80)
00000100:00000040:0.0:1788576026.007487:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007491:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576026.007492:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576026.007494:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576026.007495:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393253632, offset 0
00000400:00000200:0.0:1788576026.007498:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576026.007500:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576026.007504:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576026.007506:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576026.007507:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576026.007509:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576026.007510:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576026.007512:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331237:331237:256:4294967295] 192.168.202.110@tcp LPNI seq info [331194:331194:8:4294967295]
00000400:00000200:0.0:1788576026.007515:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576026.007520:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576026.007523:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576026.007525:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576026.007528:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576026.007530:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788576026.007533:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576026.007535:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788576026.007537:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576026.007539:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576026.007541:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576026.007543:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576026.007545:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576026.007546:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576026.007548:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576026.007550:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007555:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576026.007557:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007561:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576026.007562:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576026.007564:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576026.007566:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576026.007568:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576026.007570:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307629608
00000100:00000001:0.0:1788576026.007572:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307630522
00000100:00000001:0.0:1788576026.007573:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307629608
00000100:00000001:0.0:1788576026.007574:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307630522
00000100:00000001:0.0:1788576026.007576:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576026.007577:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576026.007578:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576026.007580:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576026/real 1788576026]  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576026.007588:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576026.007589:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576026.007590:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576026.007591:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576026.007592:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576026.007593:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576026.007595:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576026.007599:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007603:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576026.007604:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007605:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576026.007606:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576026.007609:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007616:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576026.007619:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007626:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576026.007628:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007629:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576026.007630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007632:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576026.007634:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576026.007636:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576026.007639:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576026.007642:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576026.007646:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576026.007654:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393253632:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576026.007657:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576026.007661:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007665:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576026.007666:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576026.007667:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576026.007668:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576026.007669:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576026.007671:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393253760:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576026.007673:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576026.007674:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576026.007675:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576026.007676:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576026.007679:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576026.007680:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576026.007682:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576026.007684:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576026.007686:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576026.007688:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393253760, portal 4
00000100:00000001:0.0:1788576026.007689:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576026.007690:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345465856 : -130483364085760 : ffff8953796f4000)
00000100:00000040:0.0:1788576026.007694:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007698:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576026.007699:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576026.007700:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576026.007702:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393253760, offset 0
00000400:00000200:0.0:1788576026.007704:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576026.007707:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576026.007709:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576026.007711:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576026.007713:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576026.007714:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576026.007715:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576026.007717:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331238:331238:256:4294967295] 192.168.202.110@tcp LPNI seq info [331195:331195:8:4294967295]
00000400:00000200:0.0:1788576026.007719:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576026.007723:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576026.007726:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576026.007728:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576026.007730:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576026.007732:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788576026.007734:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576026.007736:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788576026.007738:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576026.007739:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576026.007741:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576026.007743:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576026.007745:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576026.007746:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576026.007747:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576026.007749:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576051 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007754:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576026.007757:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576051 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007762:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576026.007764:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576026.007766:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576026.007769:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576026.007772:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576026.007774:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307629754
00000100:00000001:0.0:1788576026.007776:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307630522
00000100:00000001:0.0:1788576026.007778:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307629754
00000100:00000001:0.0:1788576026.007780:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307630522
00000100:00000001:0.0:1788576026.007783:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576026.007785:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576026.007786:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576026.007790:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576026/real 1788576026]  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576026.007798:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576026.007799:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576026.007801:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576026.007804:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576026.007805:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576026.007806:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576026.007808:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576026.007813:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007817:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576026.007818:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007819:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576026.007820:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576026.007822:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007825:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576026.007827:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007831:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576026.007833:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007834:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576026.007835:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007836:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576026.007837:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576026.007839:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576026.007840:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576026.007841:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576026.007843:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576026.007847:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393253760:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576026.007850:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007851:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576026.007854:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5f80 x1875444393253632/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007860:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576026.007861:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576026.007863:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576026.007865:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576026.007867:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576026.007868:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576026.007870:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576026.007871:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:0.0:1788576026.007874:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576026.007875:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576026.007876:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5f80.
00000100:00000001:0.0:1788576026.007879:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576026.007880:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007882:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576026.007884:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4000 x1875444393253760/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576051 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576026.007893:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576026.007894:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576026.007895:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576026.007897:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576026.007898:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576026.007899:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576026.007901:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576026.007908:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:0.0:1788576026.007910:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576026.007911:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576026.007912:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4000.
00000100:00000001:0.0:1788576026.007914:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576026.007915:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007917:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.007920:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576026.007921:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576026.007923:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576026.007924:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576026.390425:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576026.390433:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576026.390439:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576026.390440:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576026.390443:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576026.390445:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576026.390548:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576026.390551:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576026.390554:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576026.517539:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576026.517546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576026.517549:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576026.517554:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576026.517558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576026.517561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.517564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576026.517566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576026.517569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576026.965523:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307630522, at 4307630528
00000100:00000010:0.0:1788576026.965538:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000040:1.0:1788576026.966407:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307630522, at 4307630528
00000100:00000010:1.0:1788576026.966416:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7540.
00010000:00000001:3.0:1788576027.413505:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576027.413511:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576027.413517:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576027.413520:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576027.413523:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576027.413526:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576027.413529:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576027.413531:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576027.413534:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576027.413536:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576027.413551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576027.413553:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576027.413554:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576027.413555:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576027.413557:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576027.413559:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576027.413561:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576027.413562:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576027.413715:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576027.413719:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576027.413722:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576027.541690:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576027.541697:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576027.541699:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576027.541705:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576027.541708:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576027.541712:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576027.541714:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576027.541725:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576027.541728:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576028.437529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576028.437537:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576028.437551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576028.437553:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576028.437557:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576028.437559:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576028.437730:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576028.437734:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576028.437737:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576028.565527:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576028.565535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576028.565537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576028.565543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576028.565546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576028.565550:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576028.565553:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576028.565555:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576028.565558:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576029.461525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576029.461533:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576029.461537:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576029.461539:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576029.461544:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576029.461547:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576029.461549:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576029.461551:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576029.461554:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576029.461556:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576029.461559:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576029.461560:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576029.461626:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576029.461629:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576029.461631:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576029.589521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576029.589528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576029.589530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576029.589535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576029.589539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576029.589543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576029.589546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576029.589548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576029.589551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576030.488684:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576030.488692:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576030.488697:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576030.488699:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576030.488702:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576030.488703:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576030.488763:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576030.488767:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576030.488771:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576030.613554:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576030.613566:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576030.613568:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576030.613574:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576030.613578:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576030.613582:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576030.613584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576030.613587:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576030.613590:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576031.125506:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576031.125523:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788576031.125526:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576031.125530:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576031.125535:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576031.125542:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12962
00000400:00000200:1.0:1788576031.125550:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576031.125556:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576031.125559:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576031.125562:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576031.125564:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576031.125567:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576031.125569:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576031.125573:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576031.125575:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788576031.125578:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576031.125581:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788576031.125583:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788576031.125585:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788576031.125588:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576031.125590:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576031.125592:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576031.125594:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576031.125597:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788576031.125602:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576031.125605:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576031.125607:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576031.125610:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576031.125613:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576031.125615:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788576031.125618:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576031.125636:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788576031.125639:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788576031.125643:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576031.125644:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576031.125647:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576031.125650:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576031.125653:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576031.125660:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788576031.125667:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576031.125675:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576031.125685:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576031.125690:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576031.125694:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576031.125714:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576031.125717:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576031.125721:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788576031.125722:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576031.125724:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576031.125727:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576031.125731:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12962
00000400:00000200:1.0:1788576031.125735:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576031.125739:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:1.0:1788576031.125739:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576031.125744:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1788576031.125745:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576031.125749:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788576031.125750:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576031.125754:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576031.125758:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000040:0.0:1788576031.125760:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576031.125760:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576031.125764:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576031.125767:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788576031.125770:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576031.125773:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788576031.125778:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788576031.125779:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:1.0:1788576031.125781:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788576031.125783:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576031.125787:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:0.0:1788576031.125789:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1788576031.125790:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1788576031.125792:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1788576031.125794:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
02000000:00000001:0.0:1788576031.125796:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576031.125799:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00100000:0.0:1788576031.125802:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393253888:192.168.202.110@tcp:250:kworker.0
00000020:00000040:1.0:1788576031.125803:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788576031.125808:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:1.0:1788576031.125810:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1788576031.125813:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1788576031.125813:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576031.125816:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576031.125817:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576031.125820:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1788576031.125823:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:0.0:1788576031.125825:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:1.0:1788576031.125829:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576031.125830:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576031.125835:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:0.0:1788576031.125839:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:1.0:1788576031.125840:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000400:00000010:0.0:1788576031.125842:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000008:00000001:1.0:1788576031.125844:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576031.125847:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576031.125849:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000010:1.0:1788576031.125851:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000100:00000200:0.0:1788576031.125855:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393253888, portal 25
00000020:00000040:1.0:1788576031.125855:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788576031.125858:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1788576031.125859:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576031.125862:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345466752 : -130483364084864 : ffff8953796f4380)
02000000:00000001:1.0:1788576031.125862:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576031.125868:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:0.0:1788576031.125871:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788576031.125872:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576031.125875:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576031.125880:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:0.0:1788576031.125881:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576031.125886:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788576031.125886:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576031.125889:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576031.125892:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393253888, offset 0
00000100:00100000:1.0:1788576031.125896:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788576031.125900:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788576031.125908:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576031.125911:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576031.125915:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788576031.125917:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576031.125925:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:1.0:1788576031.125930:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576031.125931:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1788576031.125936:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788576031.125937:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576031.125942:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:1.0:1788576031.125943:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1788576031.125946:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:1.0:1788576031.125948:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1788576031.125952:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331239:331239:256:4294967295] 192.168.202.110@tcp LPNI seq info [331196:331196:8:4294967295]
00000100:00080000:1.0:1788576031.125952:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1788576031.125958:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788576031.125959:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576031.125963:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12962
00000400:00000200:0.0:1788576031.125970:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576031.125977:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:1.0:1788576031.125978:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:0.0:1788576031.125981:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000400:00000200:1.0:1788576031.125985:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:0.0:1788576031.125990:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:1.0:1788576031.125990:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000800:00000200:0.0:1788576031.125993:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:1.0:1788576031.125997:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000800:00000200:0.1:1788576031.126001:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788576031.126003:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1788576031.126004:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:1.0:1788576031.126009:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000100:0.0:1788576031.126011:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576031.126015:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000100:00000001:1.0:1788576031.126015:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576031.126021:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576031.126022:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:0.0:1788576031.126023:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:1.0:1788576031.126027:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1788576031.126029:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576031.126033:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576031.126035:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576031.126038:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576031.126042:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788576031.126042:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788576031.126048:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788576031.126054:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:0.0:1788576031.126059:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788576031.126059:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576031.126062:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576031.126067:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576056 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788576031.126069:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576031.126074:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576031.126078:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:0.0:1788576031.126084:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576031.126084:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576031.126089:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:1.0:1788576031.126090:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576031.126094:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788576031.126097:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576031.126098:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576031.126103:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576031.126107:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1788576031.126108:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576031.126110:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307635641
00000008:00100000:1.0:1788576031.126112:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00000010:0.0:1788576031.126115:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000008:00000001:1.0:1788576031.126116:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576031.126120:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576031.126123:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000010:1.0:1788576031.126125:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000100:00000001:0.0:1788576031.126130:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307635641
00000100:00000010:0.0:1788576031.126134:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ea0.
00000100:00000001:0.0:1788576031.126137:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576031.126141:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1788576031.126144:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000001:0.0:1788576031.126145:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576031.126149:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1788576031.126150:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
02000000:00000001:1.0:1788576031.126153:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576031.126157:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:0.0:1788576031.126158:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576031/real 1788576031]  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1788576031.126160:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576031.126163:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576031.126170:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000400:00000001:0.0:1788576031.126173:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1788576031.126175:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576031.126182:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:0.0:1788576031.126183:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576031.126187:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576031.126191:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576031.126193:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:1.0:1788576031.126196:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576031.126197:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000001:1.0:1788576031.126201:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576031.126205:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788576031.126210:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576031.126229:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576031.126230:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788576031.126233:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788576031.126241:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:0.0:1788576031.126246:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576031.126249:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1788576031.126249:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12972)
00000100:00000001:0.0:1788576031.126252:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576031.126254:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576031.126258:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126262:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576031.126266:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126270:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576031.126272:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126274:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576031.126274:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126276:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576031.126277:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576031.126279:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576031.126282:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576031.126284:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576031.126286:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576031.126291:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393253888:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576031.126294:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126296:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576031.126298:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4380 x1875444393253888/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126301:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576031.126303:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576031.126305:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576031.126307:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576031.126308:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576031.126309:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576031.126311:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576031.126313:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:0.0:1788576031.126315:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576031.126317:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576031.126318:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4380.
00000100:00000001:0.0:1788576031.126326:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576031.126327:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126330:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126336:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576031.126337:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576031.126339:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576031.126339:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576031.126341:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576031.126343:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126347:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576031.126348:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576031.126350:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576031.126351:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576031.126352:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576031.126354:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254016:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576031.126356:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576031.126357:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576031.126358:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576031.126359:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576031.126361:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576031.126363:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576031.126365:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576031.126367:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576031.126369:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576031.126371:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393254016, portal 10
00000100:00000001:0.0:1788576031.126372:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576031.126373:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345468544 : -130483364083072 : ffff8953796f4a80)
00000100:00000040:0.0:1788576031.126378:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126381:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576031.126383:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576031.126384:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576031.126386:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393254016, offset 0
00000400:00000200:0.0:1788576031.126388:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576031.126408:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576031.126412:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576031.126415:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576031.126417:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576031.126420:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576031.126422:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576031.126425:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331240:331240:256:4294967295] 192.168.202.110@tcp LPNI seq info [331197:331197:8:4294967295]
00000400:00000200:0.0:1788576031.126428:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576031.126434:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576031.126439:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576031.126443:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576031.126447:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576031.126463:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788576031.126468:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576031.126470:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1788576031.126474:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576031.126475:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576031.126478:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576031.126481:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576031.126484:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576031.126486:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576031.126488:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576031.126491:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126499:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576031.126502:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576056 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126507:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576031.126508:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576031.126511:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576031.126514:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576031.126517:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576031.126519:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307634727
00000100:00000001:0.0:1788576031.126522:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307635641
00000100:00000001:0.0:1788576031.126524:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307634727
00000100:00000001:0.0:1788576031.126526:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307635641
00000100:00000001:0.0:1788576031.126528:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576031.126530:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576031.126531:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576031.126535:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576031/real 1788576031]  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576031.126542:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576031.126543:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576031.126544:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576031.126546:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576031.126547:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576031.126548:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576031.126550:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576031.126555:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126559:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576031.126560:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126561:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576031.126561:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576031.126563:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126567:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576031.126569:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126573:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576031.126574:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126575:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576031.126576:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126577:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576031.126578:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576031.126580:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576031.126581:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576031.126583:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576031.126585:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576031.126598:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254016:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576031.126601:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576031.126603:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126608:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576031.126610:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576031.126612:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576031.126613:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576031.126615:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576031.126617:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576031.126620:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576031.126622:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576031.126623:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576031.126625:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576031.126628:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788576031.126630:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576031.126633:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576031.126636:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576031.126639:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576031.126642:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393254144, portal 4
00000100:00000001:0.0:1788576031.126645:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576031.126646:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345476608 : -130483364075008 : ffff8953796f6a00)
00000100:00000040:0.0:1788576031.126652:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126657:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576031.126658:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576031.126660:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576031.126661:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393254144, offset 0
00000400:00000200:0.0:1788576031.126664:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576031.126667:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576031.126669:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576031.126671:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576031.126673:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576031.126674:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576031.126675:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576031.126677:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331241:331241:256:4294967295] 192.168.202.110@tcp LPNI seq info [331198:331198:8:4294967295]
00000400:00000200:0.0:1788576031.126680:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576031.126684:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576031.126687:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576031.126689:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576031.126691:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576031.126693:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788576031.126695:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576031.126697:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1788576031.126699:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576031.126700:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576031.126702:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576031.126704:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576031.126706:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576031.126707:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576031.126708:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576031.126711:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576056 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126715:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576031.126717:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576056 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126721:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576031.126722:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576031.126723:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576031.126725:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576031.126727:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576031.126728:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307634873
00000100:00000001:0.0:1788576031.126730:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307635641
00000100:00000001:0.0:1788576031.126732:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307634873
00000100:00000001:0.0:1788576031.126734:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307635641
00000100:00000001:0.0:1788576031.126736:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576031.126738:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576031.126740:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576031.126743:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576031/real 1788576031]  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576031.126750:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576031.126752:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576031.126753:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576031.126756:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576031.126757:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576031.126759:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576031.126763:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576031.126771:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126778:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576031.126779:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126780:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576031.126781:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576031.126783:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126787:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576031.126790:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126793:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576031.126795:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126796:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576031.126797:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126798:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576031.126799:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576031.126801:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576031.126803:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576031.126804:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576031.126806:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576031.126810:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576031.126812:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126813:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576031.126815:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4a80 x1875444393254016/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126819:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576031.126820:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576031.126821:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576031.126823:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576031.126823:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576031.126824:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576031.126826:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576031.126827:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:0.0:1788576031.126828:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576031.126829:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576031.126830:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4a80.
00000100:00000001:0.0:1788576031.126831:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576031.126832:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126833:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576031.126835:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6a00 x1875444393254144/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576056 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576031.126842:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576031.126843:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576031.126844:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6400.
02000000:00000001:0.0:1788576031.126845:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576031.126846:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576031.126846:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576031.126847:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576031.126848:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:0.0:1788576031.126849:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576031.126850:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576031.126850:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6a00.
00000100:00000001:0.0:1788576031.126852:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576031.126852:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126853:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.126855:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576031.126856:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576031.126857:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576031.126858:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576031.509558:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576031.509564:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576031.509570:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576031.509571:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576031.509574:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576031.509575:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576031.509643:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576031.509647:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576031.509650:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576031.637530:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576031.637537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576031.637539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576031.637544:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576031.637548:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576031.637552:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.637554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576031.637557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576031.637560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576032.085525:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307635641, at 4307635648
00000100:00000040:0.0:1788576032.085526:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307635641, at 4307635648
00000100:00000010:0.0:1788576032.085543:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:1.0:1788576032.085545:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ea0.
00010000:00000001:3.0:1788576032.533931:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576032.533938:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576032.533944:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576032.533945:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576032.533947:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576032.533949:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576032.534118:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576032.534122:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576032.534125:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576032.661543:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576032.661552:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576032.661554:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576032.661568:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576032.661573:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576032.661576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576032.661579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576032.661582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576032.661585:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576033.557559:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576033.557564:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576033.557568:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576033.557568:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576033.557570:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576033.557571:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576033.557665:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576033.557667:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576033.557669:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576033.685598:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576033.685606:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576033.685608:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576033.685614:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576033.685617:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576033.685621:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576033.685623:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576033.685626:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576033.685628:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576034.582674:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576034.582680:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576034.582685:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576034.582686:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576034.582690:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576034.582691:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576034.582751:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576034.582755:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576034.582759:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576034.709597:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576034.709603:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576034.709606:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576034.709612:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576034.709617:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576034.709621:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576034.709623:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576034.709626:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576034.709630:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576035.605516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576035.605522:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576035.605525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576035.605526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576035.605529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576035.605530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576035.605603:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576035.605606:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576035.605608:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576035.733481:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576035.733488:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576035.733490:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576035.733495:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576035.733499:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576035.733502:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576035.733505:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576035.733507:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576035.733510:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576036.245575:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576036.245584:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576036.245587:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576036.245590:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576036.245594:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576036.245601:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12967
00000400:00000200:3.0:1788576036.245621:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576036.245627:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576036.245629:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576036.245632:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576036.245635:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576036.245637:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576036.245639:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576036.245649:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576036.245652:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576036.245655:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576036.245658:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576036.245661:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576036.245662:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576036.245665:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576036.245667:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576036.245668:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576036.245670:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576036.245673:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576036.245675:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576036.245677:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576036.245680:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576036.245683:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576036.245686:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576036.245687:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576036.245690:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576036.245698:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788576036.245702:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576036.245705:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576036.245707:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576036.245710:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576036.245713:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576036.245717:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576036.245723:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788576036.245729:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576036.245737:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576036.245746:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576036.245749:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576036.245753:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576036.245776:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576036.245779:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576036.245782:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576036.245784:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576036.245785:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576036.245788:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576036.245790:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12967
00000400:00000200:3.0:1788576036.245794:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576036.245797:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576036.245800:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576036.245804:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576036.245806:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576036.245809:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576036.245810:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788576036.245812:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576036.245818:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1788576036.245820:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1788576036.245821:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576036.245825:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788576036.245827:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1788576036.245829:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788576036.245833:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576036.245836:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788576036.245844:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788576036.245844:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576036.245849:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576036.245856:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576036.245862:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576036.245865:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788576036.245868:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576036.245879:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576036.245890:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788576036.245891:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576036.245895:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576036.245896:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576036.245901:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576036.245903:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788576036.245903:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1788576036.245908:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:3.0:1788576036.245909:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788576036.245917:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1788576036.245918:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576036.245926:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576036.245932:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393254272:192.168.202.110@tcp:250:kworker.0
00000100:00000010:3.0:1788576036.245934:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000020:00000040:3.0:1788576036.245939:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:0.0:1788576036.245946:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576036.245946:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576036.245949:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576036.245952:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576036.245954:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576036.245955:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576036.245959:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576036.245960:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576036.245967:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788576036.245968:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1788576036.245974:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576036.245980:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
00000100:00100000:3.0:1788576036.245984:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1788576036.245985:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576036.245998:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1788576036.246000:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576036.246006:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576036.246010:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000100:00000040:3.0:1788576036.246013:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788576036.246022:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000001:3.0:1788576036.246025:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576036.246030:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:0.0:1788576036.246033:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393254272, portal 25
00000100:00080000:3.0:1788576036.246035:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576036.246040:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576036.246043:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000001:3.0:1788576036.246049:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576036.246057:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:0.0:1788576036.246058:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576036.246065:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576036.246069:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12967
00000400:00000200:3.0:1788576036.246082:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576036.246090:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1788576036.246094:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1788576036.246095:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576036.246104:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1788576036.246104:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:0.0:1788576036.246110:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000001:3.0:1788576036.246110:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000200:0.0:1788576036.246116:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393254272, offset 0
00000400:00000200:0.0:1788576036.246124:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576036.246134:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576036.246143:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576036.246150:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576036.246154:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576036.246158:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576036.246162:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576036.246167:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331242:331242:256:4294967295] 192.168.202.110@tcp LPNI seq info [331199:331199:8:4294967295]
00000400:00000200:0.0:1788576036.246171:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576036.246182:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576036.246189:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576036.246196:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576036.246204:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576036.246209:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788576036.246216:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576036.246220:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1788576036.246226:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576036.246229:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576036.246237:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576036.246242:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576036.246249:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576036.246252:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576036.246257:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576036.246261:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246272:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576036.246275:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576061 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788576036.246281:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576036.246287:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576036.246289:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788576036.246293:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000001:3.0:1788576036.246295:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576036.246299:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576036.246301:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788576036.246303:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788576036.246304:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788576036.246311:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788576036.246316:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788576036.246317:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576036.246323:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307640761
00000100:00000040:3.0:1788576036.246327:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000010:0.0:1788576036.246333:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:3.0:1788576036.246334:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788576036.246345:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:3.0:1788576036.246346:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576036.246351:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788576036.246354:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307640761
00000020:00000040:3.0:1788576036.246355:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000010:0.0:1788576036.246363:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7120.
00000100:00080000:3.0:1788576036.246363:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576036.246371:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788576036.246372:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576036.246379:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788576036.246380:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576036.246383:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576036.246387:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576036.246388:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576036.246426:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576036.246428:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788576036.246430:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576036.246432:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576036.246437:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000020:00000040:3.0:1788576036.246440:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788576036.246448:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576036.246450:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576036.246451:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576036.246453:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1788576036.246456:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1788576036.246456:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576036.246465:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000100:00100000:0.0:1788576036.246467:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576036/real 1788576036]  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576036.246472:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576036.246485:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:0.0:1788576036.246486:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576036.246491:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576036.246495:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576036.246504:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576036.246509:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576036.246517:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000001:3.0:1788576036.246520:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576036.246525:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576036.246526:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788576036.246535:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576036.246551:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576036.246556:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576036.246566:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1788576036.246569:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000040:3.0:1788576036.246574:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:0.0:1788576036.246575:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.246581:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:3.0:1788576036.246584:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12977)
00000100:00000001:0.0:1788576036.246585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576036.246593:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246602:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576036.246621:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246633:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576036.246636:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.246638:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576036.246639:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.246642:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576036.246645:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576036.246654:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576036.246659:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576036.246662:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576036.246666:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576036.246675:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393254272:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576036.246681:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.246683:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576036.246685:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393254272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246690:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576036.246692:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576036.246695:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788576036.246698:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576036.246700:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576036.246701:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576036.246703:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576036.246706:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:0.0:1788576036.246709:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576036.246711:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576036.246712:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:0.0:1788576036.246715:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576036.246716:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.246718:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.246725:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576036.246726:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576036.246728:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576036.246729:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576036.246730:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576036.246733:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246738:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576036.246740:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576036.246742:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576036.246744:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576036.246747:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576036.246749:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576036.246753:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576036.246755:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576036.246757:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576036.246759:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576036.246762:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788576036.246764:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576036.246769:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576036.246772:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576036.246776:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576036.246779:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393254400, portal 10
00000100:00000001:0.0:1788576036.246781:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576036.246782:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:0.0:1788576036.246789:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246796:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576036.246798:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576036.246800:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576036.246802:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393254400, offset 0
00000400:00000200:0.0:1788576036.246807:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576036.246811:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576036.246815:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576036.246817:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576036.246820:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576036.246823:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576036.246825:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576036.246828:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331243:331243:256:4294967295] 192.168.202.110@tcp LPNI seq info [331200:331200:8:4294967295]
00000400:00000200:0.0:1788576036.246833:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576036.246840:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576036.246845:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576036.246849:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576036.246853:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576036.246856:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788576036.246860:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576036.246863:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1788576036.246867:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576036.246868:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576036.246872:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576036.246875:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576036.246878:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576036.246880:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576036.246882:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576036.246886:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246894:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576036.246896:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576061 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246903:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576036.246905:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576036.246908:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576036.246912:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576036.246916:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576036.246918:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307639847
00000100:00000001:0.0:1788576036.246921:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307640761
00000100:00000001:0.0:1788576036.246923:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307639847
00000100:00000001:0.0:1788576036.246925:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307640761
00000100:00000001:0.0:1788576036.246927:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576036.246929:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576036.246931:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576036.246935:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576036/real 1788576036]  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576036.246944:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576036.246945:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576036.246947:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576036.246949:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576036.246951:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576036.246953:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576036.246956:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576036.246963:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246970:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576036.246971:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.246973:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576036.246974:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576036.246978:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246984:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576036.246988:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.246995:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576036.246998:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.246999:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576036.247001:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.247002:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576036.247005:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576036.247007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576036.247010:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576036.247012:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576036.247016:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576036.247023:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576036.247026:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576036.247029:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.247035:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576036.247036:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576036.247039:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576036.247040:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576036.247042:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576036.247045:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254528:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576036.247048:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576036.247050:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576036.247052:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576036.247053:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576036.247057:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576036.247058:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576036.247062:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576036.247064:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576036.247067:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576036.247070:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393254528, portal 4
00000100:00000001:0.0:1788576036.247072:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576036.247074:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:0.0:1788576036.247079:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.247086:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576036.247088:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576036.247090:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576036.247092:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393254528, offset 0
00000400:00000200:0.0:1788576036.247096:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576036.247100:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576036.247103:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576036.247106:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576036.247108:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576036.247111:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576036.247113:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576036.247116:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331244:331244:256:4294967295] 192.168.202.110@tcp LPNI seq info [331201:331201:8:4294967295]
00000400:00000200:0.0:1788576036.247119:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576036.247124:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576036.247129:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576036.247132:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576036.247136:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576036.247139:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1788576036.247142:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576036.247145:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1788576036.247150:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576036.247152:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:0.0:1788576036.247155:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576036.247158:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576036.247161:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:0.0:1788576036.247162:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:0.0:1788576036.247164:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576036.247167:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576061 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.247172:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576036.247174:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576061 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.247178:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576036.247179:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576036.247181:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576036.247183:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576036.247185:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576036.247187:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307639993
00000100:00000001:0.0:1788576036.247189:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307640761
00000100:00000001:0.0:1788576036.247190:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307639993
00000100:00000001:0.0:1788576036.247192:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307640761
00000100:00000001:0.0:1788576036.247193:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576036.247195:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576036.247196:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576036.247198:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576036/real 1788576036]  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576036.247202:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576036.247203:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576036.247204:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576036.247205:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576036.247206:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576036.247207:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576036.247209:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576036.247214:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.247218:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576036.247219:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.247219:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576036.247220:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576036.247222:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.247225:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576036.247227:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.247231:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576036.247233:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.247234:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576036.247235:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.247236:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576036.247237:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576036.247239:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576036.247241:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576036.247242:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576036.247244:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576036.247248:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254528:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576036.247250:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.247251:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576036.247253:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393254400/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.247259:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576036.247261:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576036.247263:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788576036.247265:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576036.247267:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576036.247268:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576036.247270:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576036.247272:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788576036.247274:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576036.247276:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576036.247277:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:0.0:1788576036.247279:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576036.247281:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.247282:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576036.247285:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393254528/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576061 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576036.247296:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576036.247297:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576036.247298:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576036.247300:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576036.247301:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576036.247302:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576036.247304:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576036.247305:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:0.0:1788576036.247307:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576036.247308:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576036.247309:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:0.0:1788576036.247312:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576036.247313:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.247315:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.247318:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576036.247319:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576036.247321:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576036.247322:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576036.629635:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576036.629644:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576036.629652:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576036.629653:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576036.629658:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576036.629659:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576036.629912:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576036.629917:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576036.629921:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576036.758572:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576036.758580:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576036.758582:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576036.758587:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576036.758590:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576036.758594:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.758597:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576036.758599:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576036.758602:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576037.205533:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307640761, at 4307640768
00000100:00000040:1.0:1788576037.205533:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307640761, at 4307640768
00000100:00000010:0.0:1788576037.205552:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:1.0:1788576037.205556:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7120.
00010000:00000001:3.0:1788576037.654528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576037.654537:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576037.654545:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576037.654549:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576037.654552:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576037.654556:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576037.654560:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576037.654563:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576037.654565:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576037.654568:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576037.654572:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576037.654573:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576037.654574:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576037.654576:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576037.654577:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576037.654578:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576037.654580:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576037.654582:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576037.654644:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576037.654658:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576037.654662:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576037.781593:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576037.781604:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576037.781606:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576037.781613:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576037.781618:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576037.781622:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576037.781626:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576037.781630:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576037.781633:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576038.678620:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576038.678627:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576038.678632:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576038.678633:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576038.678638:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576038.678639:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576038.678722:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576038.678726:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576038.678729:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576038.805547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576038.805555:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576038.805557:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576038.805563:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576038.805566:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576038.805570:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576038.805572:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576038.805574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576038.805575:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576039.701533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576039.701542:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576039.701545:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576039.701548:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576039.701553:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576039.701555:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576039.701558:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576039.701560:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576039.701564:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576039.701565:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576039.701569:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576039.701570:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576039.701673:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576039.701676:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576039.701679:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576039.829772:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576039.829783:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576039.829785:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576039.829793:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576039.829797:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576039.829802:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576039.829806:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576039.829809:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576039.829813:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576040.725610:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576040.725616:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576040.725621:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576040.725622:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576040.725626:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576040.725627:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576040.725686:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576040.725690:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576040.725694:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576040.854074:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576040.854083:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576040.854086:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576040.854093:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576040.854097:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576040.854102:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576040.854105:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576040.854108:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576040.854111:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576041.365518:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576041.365529:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576041.365531:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576041.365534:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576041.365538:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576041.365547:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12972
00000400:00000200:3.0:1788576041.365557:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576041.365563:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576041.365566:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576041.365570:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365573:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576041.365576:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365579:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576041.365582:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576041.365584:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576041.365586:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576041.365588:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576041.365590:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576041.365591:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576041.365593:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365595:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576041.365596:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365597:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576041.365599:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576041.365601:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365602:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576041.365604:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576041.365607:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576041.365609:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576041.365610:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576041.365612:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576041.365620:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788576041.365622:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576041.365624:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576041.365626:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576041.365628:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576041.365630:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576041.365633:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576041.365638:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8c00.
00000100:00000001:3.0:1788576041.365643:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576041.365650:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576041.365657:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576041.365660:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576041.365663:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576041.365684:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576041.365686:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576041.365688:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576041.365690:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576041.365691:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576041.365692:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576041.365694:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12972
00000400:00000200:3.0:1788576041.365697:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576041.365712:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576041.365714:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576041.365715:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365717:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576041.365718:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365719:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576041.365721:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576041.365721:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576041.365722:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576041.365723:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788576041.365724:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576041.365726:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788576041.365727:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365728:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576041.365729:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365731:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576041.365732:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788576041.365734:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365735:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576041.365736:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576041.365740:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576041.365742:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576041.365743:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576041.365746:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788576041.365748:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576041.365749:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576041.365752:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788576041.365753:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788576041.365754:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576041.365755:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576041.365756:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576041.365757:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576041.365763:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576041.365766:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfbc00.
00000100:00000001:3.0:1788576041.365769:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576041.365772:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576041.365777:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576041.365778:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576041.365780:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576041.365785:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576041.365787:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576041.365789:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788576041.365790:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576041.365790:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576041.365792:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576041.365793:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12972
00000400:00000200:3.0:1788576041.365795:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576041.365797:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576041.365799:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576041.365801:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365802:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576041.365803:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365804:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576041.365806:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576041.365806:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576041.365807:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576041.365808:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788576041.365809:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788576041.365810:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788576041.365812:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365813:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576041.365814:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365815:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576041.365816:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788576041.365817:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576041.365818:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576041.365819:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576041.365821:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576041.365822:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576041.365823:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576041.365824:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788576041.365825:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576041.365826:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576041.365828:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000020:00000040:3.0:1788576041.365829:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788576041.365830:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576041.365831:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576041.365831:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576041.365832:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576041.365833:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576041.365836:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa800.
00000100:00000001:3.0:1788576041.365839:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576041.365843:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576041.365850:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576041.365852:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576041.365855:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576041.365862:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576041.365864:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788576041.365867:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788576041.365884:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12982)
00000100:00000001:0.0:1788576041.366139:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576041.366147:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576041.366151:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576041.366165:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576041.366176:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576041.366179:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576041.366184:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576041.366187:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576041.366190:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576041.366194:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393254656:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576041.366199:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576041.366204:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576041.366207:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576041.366210:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576041.366220:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7c00.
02000000:00000001:0.0:1788576041.366222:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576041.366230:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576041.366234:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576041.366243:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576041.366251:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393254656, portal 25
00000100:00000001:0.0:1788576041.366254:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576041.366256:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:0.0:1788576041.366262:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576041.366270:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576041.366273:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576041.366275:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4cc0.
00000100:00000200:0.0:1788576041.366278:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393254656, offset 0
00000400:00000200:0.0:1788576041.366285:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576041.366295:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576041.366303:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576041.366310:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576041.366314:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576041.366319:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576041.366322:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576041.366326:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331245:331245:256:4294967295] 192.168.202.110@tcp LPNI seq info [331202:331202:8:4294967295]
00000400:00000200:0.0:1788576041.366331:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576041.366341:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576041.366348:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576041.366355:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9500.
00000800:00000200:0.0:1788576041.366362:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576041.366367:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788576041.366430:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576041.366436:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576041.366440:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307645882
00000100:00000010:0.0:1788576041.366446:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576041.366454:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576041.366461:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307645882
00000100:00000010:0.0:1788576041.366465:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7120.
00000100:00000001:0.0:1788576041.366469:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576041.366474:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576041.366477:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576041.366480:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576041.366485:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576041.366494:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576041.366495:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576041.366498:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576041.366500:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576041.366502:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576041.366505:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254784:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576041.366509:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576041.366512:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576041.366513:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576041.366515:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576041.366519:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576041.366521:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576041.366525:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576041.366537:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da580.
00000400:00000010:0.0:1788576041.366545:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576041.366549:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393254784, portal 10
00000100:00000001:0.0:1788576041.366551:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576041.366553:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:0.0:1788576041.366561:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576041.366568:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576041.366571:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576041.366574:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d44c8.
00000100:00000200:0.0:1788576041.366577:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393254784, offset 0
00000400:00000200:0.0:1788576041.366581:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576041.366585:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576041.366589:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576041.366592:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576041.366594:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576041.366597:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576041.366599:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576041.366602:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331246:331246:255:4294967295] 192.168.202.110@tcp LPNI seq info [331203:331203:7:4294967295]
00000400:00000200:0.0:1788576041.366607:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576041.366615:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576041.366620:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576041.366624:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9200.
00000800:00000200:0.0:1788576041.366628:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576041.366632:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788576041.366635:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576041.366639:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576041.366642:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576041.366644:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307644967
00000100:00000001:0.0:1788576041.366647:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307645882
00000100:00000001:0.0:1788576041.366650:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307644967
00000100:00000001:0.0:1788576041.366652:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307645882
00000100:00000001:0.0:1788576041.366654:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576041.366656:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576041.366659:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576041.366663:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576041.366671:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576041.366672:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576041.366675:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576041.366677:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576041.366679:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576041.366681:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254912:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576041.366685:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576041.366687:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576041.366689:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576041.366691:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576041.366694:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576041.366696:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576041.366713:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576041.366716:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576041.366721:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576041.366724:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393254912, portal 4
00000100:00000001:0.0:1788576041.366726:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576041.366727:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:0.0:1788576041.366733:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576041.366740:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576041.366743:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576041.366745:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4e58.
00000100:00000200:0.0:1788576041.366748:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393254912, offset 0
00000400:00000200:0.0:1788576041.366752:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576041.366756:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576041.366759:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576041.366762:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576041.366765:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576041.366767:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576041.366770:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576041.366773:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331247:331247:254:4294967295] 192.168.202.110@tcp LPNI seq info [331204:331204:6:4294967295]
00000400:00000200:0.0:1788576041.366777:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576041.366783:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576041.366789:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576041.366793:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9300.
00000800:00000200:0.0:1788576041.366797:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576041.366800:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788576041.366803:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576041.366811:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576041.366818:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576041.366820:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307645113
00000100:00000001:0.0:1788576041.366823:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307645882
00000100:00000001:0.0:1788576041.366825:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307645113
00000100:00000001:0.0:1788576041.366827:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307645882
00000100:00000001:0.0:1788576041.366829:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576041.366831:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576041.366834:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576041.366837:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576041.366847:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576041.366849:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788576041.366852:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576041.366853:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576041.366855:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576041.366857:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576041.749538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576041.749545:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576041.749551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576041.749553:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576041.749557:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576041.749558:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576041.749626:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576041.749633:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576041.749639:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576041.877552:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576041.877560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576041.877563:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576041.877569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576041.877573:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576041.877577:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576041.877579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576041.877581:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576041.877584:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576042.325617:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307645882, at 4307645888
00000100:00000010:0.0:1788576042.325637:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000040:1.0:1788576042.327828:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307645882, at 4307645890
00000100:00000010:1.0:1788576042.327844:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7120.
00010000:00000001:3.0:1788576042.773715:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576042.773725:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576042.773732:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576042.773733:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576042.773737:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576042.773739:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576042.773807:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576042.773812:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576042.773816:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576042.901550:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576042.901558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576042.901563:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576042.901570:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576042.901575:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576042.901584:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576042.901587:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576042.901589:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576042.901593:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576043.797537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576043.797547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576043.797555:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576043.797556:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576043.797561:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576043.797563:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576043.797629:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576043.797633:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576043.797637:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576043.925576:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576043.925587:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576043.925589:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576043.925596:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576043.925601:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576043.925606:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576043.925609:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576043.925612:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576043.925616:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1788576044.438077:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1788576044.438133:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1788576044.438143:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576044.438148:0:2350:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1788576044.438153:0:2350:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576044.438157:0:2350:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1788576044.438163:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788576044.438167:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9500.
00000400:00000200:1.0:1788576044.438175:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788576044.438180:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788576044.438188:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4cc0
00000400:00000010:1.0:1788576044.438191:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4cc0.
00000100:00000001:1.0:1788576044.438198:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788576044.438209:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.438412:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576044.438418:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576044.438421:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1788576044.438691:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788576044.438698:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576066 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576044.440814:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576041/real 1788576044]  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576066 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576044.440848:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576044.440852:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576044.440856:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576044.440863:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576044.440866:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576044.440870:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576044.440875:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576066 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576044.440884:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576066 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.440893:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576044.440895:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.440899:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576044.440902:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576044.440906:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576066 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.440914:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576044.440920:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576066 ref 2 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.440929:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576044.440932:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.440934:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576044.440935:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.440939:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576044.440941:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576044.440946:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576044.440952:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576044.440954:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576044.440959:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576066 ref 2 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576044.440967:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393254656:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576044.440977:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576044.440982:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576044.440986:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393254656/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576066 ref 2 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.440994:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576044.440997:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576044.441000:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576044.441002:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576044.441004:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576044.441005:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576044.441777:0:2350:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1788576044.441787:0:2350:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1788576044.441794:0:2350:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7c00.
02000000:00000001:1.0:1788576044.441814:0:2350:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1788576044.441818:0:2350:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1788576044.441820:0:2350:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1788576044.441824:0:2350:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1788576044.441827:0:2350:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8c00.
02000000:00000001:1.0:1788576044.441831:0:2350:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1788576044.441834:0:2350:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1788576044.441836:0:2350:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:1.0:1788576044.441841:0:2350:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1788576044.441842:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576044.441845:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788576044.441865:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788576044.441869:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9200.
00000400:00000200:1.0:1788576044.441874:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788576044.441880:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788576044.441888:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d44c8
00000400:00000010:1.0:1788576044.441892:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d44c8.
00000100:00000001:1.0:1788576044.441896:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788576044.441904:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576044.441940:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788576044.441944:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576066 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576044.441950:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576044.441952:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:1.0:1788576044.441957:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1788576044.441959:0:2350:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9300.
00000400:00000200:1.0:1788576044.441962:0:2350:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1788576044.441965:0:2350:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1788576044.441967:0:2350:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4e58
00000400:00000010:1.0:1788576044.441969:0:2350:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4e58.
00000100:00000001:1.0:1788576044.441971:0:2350:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:1.0:1788576044.441975:0:2350:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576066 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576044.441982:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1788576044.441985:0:2350:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576066 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576044.441991:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576044.441992:0:2350:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576044.442830:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576044.442836:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576044.442838:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576044.442848:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576041/real 1788576044]  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576044.442864:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576044.442866:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576044.442870:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da580.
00000400:00000200:0.0:1788576044.442874:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576044.442877:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576044.442881:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576044.442884:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576044.442894:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.442901:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576044.442903:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.442906:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576044.442909:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576044.442913:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.442919:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576044.442925:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.442932:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576044.442936:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.442938:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576044.442939:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.442942:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576044.442946:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576044.442950:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576044.442957:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576044.442959:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576044.442964:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576044.442973:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254784:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576044.442978:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576044.442982:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576041/real 1788576044]  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576044.442989:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576044.442991:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576044.442993:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576044.442996:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576044.442998:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576044.443000:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576044.443004:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576044.443012:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.443018:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576044.443020:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.443022:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576044.443023:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576044.443026:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.443033:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576044.443036:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.443043:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576044.443045:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.443047:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576044.443048:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.443050:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576044.443052:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576044.443054:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576044.443059:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576044.443062:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576044.443066:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576044.443085:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393254912:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576044.443090:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.443092:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576044.443096:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393254784/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.443102:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576044.443104:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576044.443109:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576044.443114:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576044.443115:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576044.443118:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576044.443121:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576044.443123:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfbc00.
02000000:00000001:0.0:1788576044.443127:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576044.443129:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576044.443131:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:0.0:1788576044.443135:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576044.443136:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.443139:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576044.443142:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393254912/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576066 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576044.443148:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576044.443149:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576044.443151:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576044.443154:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576044.443155:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576044.443156:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576044.443158:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576044.443160:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa800.
02000000:00000001:0.0:1788576044.443163:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576044.443165:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576044.443166:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:0.0:1788576044.443168:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576044.443170:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.443270:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576044.443281:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576044.443284:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576044.443296:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576044.443299:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576044.821571:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576044.821586:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576044.821593:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576044.821595:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576044.821600:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576044.821602:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576044.821717:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576044.821722:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576044.821726:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576044.949563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576044.949575:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576044.949577:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576044.949584:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576044.949589:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576044.949595:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576044.949598:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576044.949601:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576044.949605:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576045.845580:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576045.845591:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576045.845598:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576045.845600:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576045.845604:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576045.845605:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576045.845778:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576045.845782:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576045.845787:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576045.973536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576045.973547:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576045.973550:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576045.973558:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576045.973563:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576045.973568:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576045.973571:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576045.973574:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576045.973577:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576046.485864:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576046.485879:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788576046.485882:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576046.485886:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576046.485892:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576046.485901:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12977
00000400:00000200:2.0:1788576046.485914:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576046.485921:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576046.485925:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576046.485929:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.485932:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576046.485935:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.485938:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576046.485944:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576046.485946:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788576046.485950:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576046.485954:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788576046.485956:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788576046.485959:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788576046.485963:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.485964:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576046.485966:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.485968:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576046.485971:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788576046.485975:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576046.485978:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576046.485982:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576046.485986:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576046.485990:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576046.485993:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788576046.485995:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576046.486004:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01880.
00000020:00000040:2.0:1788576046.486008:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788576046.486013:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576046.486015:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576046.486020:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576046.486023:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576046.486028:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576046.486037:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788576046.486046:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576046.486060:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576046.486074:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576046.486081:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576046.486087:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ab28ae01] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576046.486116:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576046.486120:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576046.486124:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788576046.486126:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576046.486128:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576046.486131:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576046.486134:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12977
00000400:00000200:2.0:1788576046.486139:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576046.486142:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576046.486144:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576046.486147:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486150:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576046.486152:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486153:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576046.486156:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576046.486157:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788576046.486159:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576046.486162:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788576046.486164:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788576046.486166:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788576046.486169:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486171:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576046.486173:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486175:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576046.486177:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788576046.486180:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486182:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576046.486184:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576046.486191:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576046.486194:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576046.486196:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576046.486202:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788576046.486205:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576046.486207:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576046.486212:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00380.
00000020:00000040:2.0:1788576046.486214:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788576046.486217:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576046.486218:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576046.486219:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576046.486221:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576046.486222:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576046.486226:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788576046.486230:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576046.486235:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576046.486241:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576046.486243:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576046.486246:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fc98b80] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576046.486253:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576046.486255:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576046.486259:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788576046.486260:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576046.486262:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576046.486264:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576046.486266:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12977
00000400:00000200:2.0:1788576046.486270:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576046.486272:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576046.486274:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576046.486277:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486279:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576046.486280:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486282:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576046.486284:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576046.486285:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788576046.486287:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576046.486289:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788576046.486291:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788576046.486293:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788576046.486295:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486297:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576046.486299:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486301:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576046.486304:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788576046.486306:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576046.486308:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576046.486310:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576046.486313:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576046.486315:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576046.486316:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576046.486319:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788576046.486320:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576046.486322:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576046.486326:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00000.
00000020:00000040:2.0:1788576046.486328:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788576046.486330:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576046.486331:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576046.486332:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576046.486334:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576046.486335:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576046.486339:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788576046.486341:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576046.486346:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576046.486352:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576046.486354:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576046.486358:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bf5936f] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576046.486365:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576046.486367:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788576046.486372:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788576046.486378:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12987)
00000100:00000001:0.0:1788576046.486589:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576046.486596:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576046.486599:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576046.486608:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.486624:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576046.486627:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576046.486634:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576046.486638:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576046.486641:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576046.486644:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393255040:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576046.486650:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576046.486657:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576046.486660:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576046.486664:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576046.486671:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576046.486673:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576046.486681:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576046.486687:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576046.486697:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576046.486705:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393255040, portal 25
00000100:00000001:0.0:1788576046.486708:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576046.486710:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530777728 : -130484178773888 : ffff895348e01880)
00000100:00000040:0.0:1788576046.486730:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.486738:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576046.486741:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576046.486745:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576046.486748:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393255040, offset 0
00000400:00000200:0.0:1788576046.486755:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576046.486766:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576046.486775:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576046.486784:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576046.486789:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576046.486794:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576046.486798:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576046.486803:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331248:331248:256:4294967295] 192.168.202.110@tcp LPNI seq info [331205:331205:8:4294967295]
00000400:00000200:0.0:1788576046.486808:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576046.486819:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576046.486827:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576046.486834:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576046.486842:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576046.486846:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788576046.486853:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576046.486857:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788576046.486861:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576046.486864:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576046.486872:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576046.486876:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576046.486884:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576046.486886:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576046.486891:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576046.486895:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.486906:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576046.486910:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.486918:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576046.486920:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576046.486928:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576046.486933:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576046.486936:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307651002
00000100:00000010:0.0:1788576046.486941:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576046.486948:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576046.486955:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307651002
00000100:00000010:0.0:1788576046.486957:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ea0.
00000100:00000001:0.0:1788576046.486960:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576046.486964:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576046.486968:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576046.486970:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576046.486975:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576046/real 1788576046]  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576046.486985:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576046.486987:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576046.486989:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576046.486992:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576046.486994:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576046.486997:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576046.487000:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576046.487008:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487014:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576046.487015:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.487018:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576046.487021:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576046.487024:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487030:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576046.487037:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487044:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576046.487047:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.487048:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576046.487049:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.487052:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576046.487055:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576046.487059:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576046.487063:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576046.487067:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576046.487072:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576046.487081:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393255040:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576046.487086:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576046.487089:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487095:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576046.487097:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576046.487099:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576046.487101:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576046.487103:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576046.487106:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576046.487109:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576046.487111:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576046.487113:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576046.487115:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576046.487118:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576046.487121:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576046.487125:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576046.487129:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576046.487133:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576046.487135:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393255168, portal 10
00000100:00000001:0.0:1788576046.487138:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576046.487139:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530772352 : -130484178779264 : ffff895348e00380)
00000100:00000040:0.0:1788576046.487148:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487155:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576046.487158:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576046.487160:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576046.487164:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393255168, offset 0
00000400:00000200:0.0:1788576046.487179:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576046.487184:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576046.487189:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576046.487192:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576046.487194:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576046.487196:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576046.487198:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576046.487202:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331249:331249:256:4294967295] 192.168.202.110@tcp LPNI seq info [331206:331206:8:4294967295]
00000400:00000200:0.0:1788576046.487206:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576046.487212:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576046.487218:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576046.487223:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576046.487227:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576046.487230:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788576046.487233:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576046.487236:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788576046.487239:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576046.487241:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576046.487244:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576046.487247:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576046.487251:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576046.487252:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576046.487255:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576046.487259:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487266:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576046.487269:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487275:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576046.487277:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576046.487280:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576046.487286:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576046.487289:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576046.487291:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307650087
00000100:00000001:0.0:1788576046.487295:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307651002
00000100:00000001:0.0:1788576046.487297:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307650087
00000100:00000001:0.0:1788576046.487299:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307651002
00000100:00000001:0.0:1788576046.487302:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576046.487304:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576046.487306:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576046.487309:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576046/real 1788576046]  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576046.487317:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576046.487319:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576046.487321:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576046.487323:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576046.487325:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576046.487328:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576046.487331:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576046.487339:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487345:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576046.487346:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.487348:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576046.487350:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576046.487353:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487359:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576046.487363:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487370:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576046.487373:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.487374:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576046.487375:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.487377:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576046.487380:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576046.487383:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576046.487386:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576046.487388:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576046.487432:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576046.487440:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576046.487445:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576046.487448:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487454:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576046.487456:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576046.487458:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576046.487460:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576046.487462:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576046.487465:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576046.487468:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576046.487470:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576046.487472:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576046.487473:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576046.487478:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576046.487480:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576046.487483:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576046.487486:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576046.487491:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576046.487494:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393255296, portal 4
00000100:00000001:0.0:1788576046.487496:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576046.487497:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530771456 : -130484178780160 : ffff895348e00000)
00000100:00000040:0.0:1788576046.487505:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487511:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576046.487513:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576046.487516:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576046.487518:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393255296, offset 0
00000400:00000200:0.0:1788576046.487522:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576046.487526:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576046.487530:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576046.487533:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576046.487535:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576046.487537:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576046.487540:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576046.487543:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331250:331250:256:4294967295] 192.168.202.110@tcp LPNI seq info [331207:331207:8:4294967295]
00000400:00000200:0.0:1788576046.487547:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576046.487553:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576046.487557:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576046.487561:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576046.487564:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576046.487566:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788576046.487570:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576046.487573:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788576046.487577:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576046.487578:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576046.487581:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576046.487585:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576046.487588:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576046.487590:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576046.487593:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576046.487596:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487603:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576046.487606:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576071 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487613:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576046.487615:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576046.487618:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576046.487621:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576046.487624:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576046.487626:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307650234
00000100:00000001:0.0:1788576046.487629:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307651002
00000100:00000001:0.0:1788576046.487631:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307650234
00000100:00000001:0.0:1788576046.487633:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307651002
00000100:00000001:0.0:1788576046.487635:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576046.487637:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576046.487639:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576046.487642:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576046/real 1788576046]  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576046.487649:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576046.487651:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576046.487652:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576046.487654:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576046.487656:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576046.487658:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576046.487660:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576046.487668:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487679:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576046.487680:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.487682:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576046.487683:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576046.487686:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487692:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576046.487695:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.487700:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576046.487703:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.487704:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576046.487706:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.487708:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576046.487709:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576046.489909:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576046.489928:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576046.489934:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576046.489947:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576046.489957:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576046.489965:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.489969:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576046.489975:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01880 x1875444393255040/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.489983:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576046.489987:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576046.489994:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576046.490000:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576046.490005:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576046.490008:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576046.490010:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576046.490014:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788576046.490019:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576046.490022:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576046.490025:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01880.
00000100:00000001:0.0:1788576046.490030:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576046.490032:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.490034:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576046.490038:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00380 x1875444393255168/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.490043:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576046.490045:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576046.490046:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576046.490048:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576046.490049:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576046.490050:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576046.490051:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576046.490052:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788576046.490054:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576046.490055:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576046.490056:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00380.
00000100:00000001:0.0:1788576046.490058:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576046.490059:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.490060:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576046.490062:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00000 x1875444393255296/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576071 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576046.490065:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576046.490065:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576046.490067:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576046.490068:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576046.490069:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576046.490070:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576046.490072:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576046.490073:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788576046.490074:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576046.490075:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576046.490076:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00000.
00000100:00000001:0.0:1788576046.490077:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576046.490078:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.490079:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576046.490089:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576046.490090:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576046.490093:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576046.490094:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576046.869581:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576046.869590:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576046.869597:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576046.869601:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576046.869604:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576046.869608:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576046.869612:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576046.869615:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576046.869617:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576046.869619:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576046.869624:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576046.869626:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576046.869627:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576046.869628:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576046.869630:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576046.869631:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576046.869641:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576046.869644:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576046.869745:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576046.869750:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576046.869754:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576046.997615:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576046.997626:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576046.997629:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576046.997637:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576046.997640:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576046.997645:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576046.997648:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576046.997651:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576046.997654:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576047.445525:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307651002, at 4307651008
00000100:00000040:1.0:1788576047.445531:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307651002, at 4307651008
00000100:00000010:0.0:1788576047.445549:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:1.0:1788576047.445554:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ea0.
00010000:00000001:3.0:1788576047.893560:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576047.893569:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576047.893576:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576047.893577:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576047.893582:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576047.893583:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576047.893645:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576047.893648:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576047.893652:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576048.021589:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576048.021599:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576048.021601:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576048.021607:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576048.021610:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576048.021614:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576048.021617:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576048.021619:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576048.021621:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576048.917514:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576048.917521:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576048.917523:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576048.917525:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576048.917529:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576048.917530:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576048.917532:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576048.917534:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576048.917536:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576048.917537:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576048.917539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576048.917540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576048.917590:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576048.917594:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576048.917597:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576049.045538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576049.045546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576049.045547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576049.045554:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576049.045558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576049.045561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576049.045564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576049.045566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576049.045569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576049.941522:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576049.941528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576049.941534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576049.941536:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576049.941539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576049.941540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576049.941937:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576049.941941:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576049.941945:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576050.069517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576050.069525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576050.069527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576050.069533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576050.069537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576050.069541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576050.069544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576050.069546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576050.069549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576050.965819:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576050.965826:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576050.965832:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576050.965834:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576050.965838:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576050.965839:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576050.966040:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576050.966044:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576050.966047:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576051.093512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576051.093520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576051.093522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576051.093528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576051.093531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576051.093535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576051.093537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576051.093539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576051.093542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576051.605540:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576051.605555:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576051.605558:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576051.605562:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576051.605568:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576051.605578:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12982
00000400:00000200:3.0:1788576051.605590:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576051.605598:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576051.605603:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576051.605607:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605609:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576051.605611:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605613:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576051.605619:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576051.605621:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576051.605624:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576051.605629:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576051.605631:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576051.605634:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576051.605637:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605639:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576051.605641:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605644:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576051.605647:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576051.605651:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605654:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576051.605657:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576051.605661:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576051.605664:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576051.605667:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576051.605669:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576051.605679:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1180.
00000020:00000040:3.0:1788576051.605682:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576051.605687:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576051.605688:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576051.605691:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576051.605694:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576051.605698:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576051.605706:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788576051.605713:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576051.605722:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576051.605744:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576051.605747:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576051.605751:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca1dca9a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576051.605772:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576051.605775:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576051.605778:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576051.605779:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576051.605780:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576051.605783:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576051.605784:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12982
00000400:00000200:3.0:1788576051.605787:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576051.605789:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576051.605791:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576051.605794:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605795:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576051.605797:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605799:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788576051.605800:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1788576051.605801:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576051.605802:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576051.605805:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788576051.605806:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1788576051.605807:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00000001:0.0:1788576051.605810:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1788576051.605810:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576051.605814:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788576051.605819:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788576051.605820:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605823:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576051.605827:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605831:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576051.605835:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:0.0:1788576051.605837:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1788576051.605839:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576051.605840:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576051.605843:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576051.605845:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576051.605845:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576051.605848:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576051.605851:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576051.605852:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1788576051.605855:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393255424:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1788576051.605856:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576051.605860:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576051.605862:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:3.0:1788576051.605865:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788576051.605867:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:3.0:1788576051.605869:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576051.605871:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576051.605872:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576051.605877:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:3.0:1788576051.605878:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2d80.
00000020:00000040:3.0:1788576051.605881:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788576051.605883:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576051.605883:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576051.605884:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576051.605885:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:0.0:1788576051.605887:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:3.0:1788576051.605887:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576051.605892:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576051.605893:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788576051.605898:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576051.605902:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1788576051.605904:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1788576051.605910:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000100:00000001:3.0:1788576051.605911:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576051.605915:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576051.605919:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000995ec758] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788576051.605922:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000001:3.0:1788576051.605926:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576051.605928:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576051.605931:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000200:0.0:1788576051.605932:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393255424, portal 25
00000100:00000001:3.0:1788576051.605932:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576051.605935:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576051.605936:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576051.605939:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818944 : -130483570732672 : ffff89536d1e1180)
00000100:00000040:0.0:1788576051.605946:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576076 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576051.605946:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576051.605953:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12982
00000400:00000200:3.0:1788576051.605956:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576051.605960:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1788576051.605962:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1788576051.605966:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576051.605968:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576051.605972:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000040:3.0:1788576051.605975:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000200:0.0:1788576051.605976:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393255424, offset 0
00000100:00000001:3.0:1788576051.605982:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576051.605986:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788576051.605990:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576051.605992:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576051.605995:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576051.605996:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576051.605997:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576051.605998:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:0.0:1788576051.606002:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1788576051.606002:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788576051.606005:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788576051.606011:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1788576051.606017:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788576051.606018:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1788576051.606020:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576051.606023:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788576051.606025:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1788576051.606027:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788576051.606030:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000040:3.0:1788576051.606031:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:0.0:1788576051.606034:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:3.0:1788576051.606037:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1788576051.606043:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331251:331251:256:4294967295] 192.168.202.110@tcp LPNI seq info [331208:331208:8:4294967295]
00000100:00000001:3.0:1788576051.606043:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576051.606047:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576051.606050:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576051.606054:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576051.606055:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576051.606057:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576051.606060:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788576051.606062:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576051.606063:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576051.606070:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000010:3.0:1788576051.606070:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0700.
00000020:00000040:3.0:1788576051.606074:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788576051.606075:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576051.606076:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576051.606078:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576051.606080:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576051.606083:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788576051.606087:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000010:3.0:1788576051.606087:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788576051.606093:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788576051.606097:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000100:00100000:3.0:1788576051.606098:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1788576051.606111:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1788576051.606114:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576051.606117:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788576051.606118:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:3.0:1788576051.606121:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fdb0dc14] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576051.606129:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788576051.606131:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788576051.606132:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788576051.606138:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:0.1:1788576051.606139:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:3.0:1788576051.606143:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12992)
00000800:00000100:0.0:1788576051.606148:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576051.606151:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576051.606158:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576051.606162:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576051.606168:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576051.606170:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576051.606174:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576051.606177:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576076 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606186:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576051.606190:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576076 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606198:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576051.606200:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576051.606207:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576051.606211:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576051.606214:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307656121
00000100:00000010:0.0:1788576051.606220:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576051.606228:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576051.606234:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307656121
00000100:00000010:0.0:1788576051.606237:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7540.
00000100:00000001:0.0:1788576051.606239:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576051.606241:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576051.606243:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576051.606245:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576051.606248:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576051/real 1788576051]  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576051.606256:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576051.606258:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576051.606260:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576051.606262:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576051.606263:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576051.606265:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576051.606267:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576051.606277:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606281:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576051.606283:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606284:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576051.606286:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576051.606288:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606292:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576051.606296:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606300:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576051.606302:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606303:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576051.606304:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606306:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576051.606308:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576051.606310:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576051.606312:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576051.606314:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576051.606316:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576051.606321:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393255424:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576051.606325:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606327:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576051.606328:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1180 x1875444393255424/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606337:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576051.606339:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576051.606342:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576051.606344:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576051.606345:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576051.606346:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576051.606348:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576051.606350:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfac00.
02000000:00000001:0.0:1788576051.606353:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576051.606355:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576051.606356:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1180.
00000100:00000001:0.0:1788576051.606358:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576051.606359:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606360:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606368:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576051.606369:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576051.606371:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576051.606372:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576051.606373:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576051.606375:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606379:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576051.606380:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576051.606381:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576051.606383:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576051.606384:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576051.606386:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255552:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576051.606387:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576051.606389:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576051.606390:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576051.606409:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576051.606411:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576051.606413:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576051.606415:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576051.606418:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576051.606420:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576051.606422:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393255552, portal 10
00000100:00000001:0.0:1788576051.606424:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576051.606424:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138826112 : -130483570725504 : ffff89536d1e2d80)
00000100:00000040:0.0:1788576051.606430:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576076 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606434:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576051.606435:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576051.606437:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576051.606438:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393255552, offset 0
00000400:00000200:0.0:1788576051.606440:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576051.606443:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576051.606445:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576051.606447:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576051.606448:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576051.606450:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576051.606451:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576051.606454:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331252:331252:256:4294967295] 192.168.202.110@tcp LPNI seq info [331209:331209:8:4294967295]
00000400:00000200:0.0:1788576051.606456:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576051.606460:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576051.606463:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576051.606465:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576051.606467:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576051.606469:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788576051.606473:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576051.606475:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788576051.606479:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576051.606480:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576051.606483:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576051.606487:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576051.606490:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576051.606491:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576051.606494:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576051.606498:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576076 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606505:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576051.606508:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576076 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606515:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576051.606517:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576051.606520:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576051.606524:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576051.606527:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576051.606528:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307655207
00000100:00000001:0.0:1788576051.606530:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307656121
00000100:00000001:0.0:1788576051.606532:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307655207
00000100:00000001:0.0:1788576051.606533:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307656121
00000100:00000001:0.0:1788576051.606534:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576051.606536:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576051.606537:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576051.606539:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576051/real 1788576051]  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576051.606543:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576051.606544:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576051.606545:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576051.606547:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576051.606548:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576051.606549:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576051.606552:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576051.606556:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606565:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576051.606565:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606567:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576051.606568:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576051.606570:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606573:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576051.606575:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606579:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576051.606580:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606581:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576051.606581:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606582:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576051.606584:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576051.606586:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576051.606588:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576051.606589:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576051.606592:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576051.606596:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255552:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576051.606598:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576051.606600:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606603:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576051.606604:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576051.606606:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576051.606607:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576051.606608:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576051.606610:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255680:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576051.606612:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576051.606614:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576051.606615:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576051.606616:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576051.606618:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788576051.606620:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576051.606624:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576051.606626:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576051.606629:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576051.606632:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393255680, portal 4
00000100:00000001:0.0:1788576051.606634:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576051.606635:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138816256 : -130483570735360 : ffff89536d1e0700)
00000100:00000040:0.0:1788576051.606642:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576076 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606648:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576051.606651:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576051.606653:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576051.606655:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393255680, offset 0
00000400:00000200:0.0:1788576051.606659:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576051.606662:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576051.606666:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576051.606668:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576051.606669:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576051.606671:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576051.606673:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576051.606675:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331253:331253:256:4294967295] 192.168.202.110@tcp LPNI seq info [331210:331210:8:4294967295]
00000400:00000200:0.0:1788576051.606677:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576051.606681:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576051.606684:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576051.606687:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576051.606689:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576051.606691:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788576051.606693:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576051.606695:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788576051.606698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576051.606699:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576051.606701:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576051.606703:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576051.606705:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576051.606706:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576051.606708:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576051.606710:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576076 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606715:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576051.606718:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576076 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606738:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576051.606741:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576051.606744:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576051.606747:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576051.606751:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576051.606753:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307655353
00000100:00000001:0.0:1788576051.606755:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307656121
00000100:00000001:0.0:1788576051.606757:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307655353
00000100:00000001:0.0:1788576051.606760:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307656121
00000100:00000001:0.0:1788576051.606762:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576051.606764:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576051.606766:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576051.606770:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576051/real 1788576051]  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576051.606779:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576051.606780:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576051.606782:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576051.606785:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576051.606786:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576051.606788:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576051.606791:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576051.606800:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606806:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576051.606808:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606810:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576051.606811:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576051.606815:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606822:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576051.606825:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606832:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576051.606834:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606836:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576051.606837:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606839:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576051.606841:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576051.606844:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576051.606847:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576051.606849:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576051.606854:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576051.606860:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255680:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576051.606862:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606863:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576051.606865:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2d80 x1875444393255552/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606869:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576051.606871:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576051.606872:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576051.606875:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576051.606876:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576051.606878:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576051.606880:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576051.606881:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb800.
02000000:00000001:0.0:1788576051.606884:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576051.606885:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576051.606886:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2d80.
00000100:00000001:0.0:1788576051.606890:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576051.606891:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606893:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576051.606896:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0700 x1875444393255680/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576076 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576051.606902:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576051.606904:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576051.606905:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5400.
02000000:00000001:0.0:1788576051.606907:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576051.606908:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576051.606909:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576051.606911:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576051.606912:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8400.
02000000:00000001:0.0:1788576051.606914:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576051.606916:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576051.606917:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0700.
00000100:00000001:0.0:1788576051.606919:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576051.606920:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606921:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576051.606923:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576051.606924:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576051.606925:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576051.606926:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576051.989556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576051.989566:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576051.989572:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576051.989573:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576051.989578:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576051.989579:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576051.989662:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576051.989667:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576051.989670:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576052.117544:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576052.117552:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576052.117554:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576052.117560:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576052.117570:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576052.117582:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576052.117585:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576052.117591:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576052.117595:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576052.565547:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307656121, at 4307656128
00000100:00000040:1.0:1788576052.565548:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307656121, at 4307656128
00000100:00000010:1.0:1788576052.565572:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7540.
00000100:00000010:0.0:1788576052.565581:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00010000:00000001:3.0:1788576053.013513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576053.013519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576053.013524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576053.013525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576053.013528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576053.013530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576053.013618:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576053.013624:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576053.013629:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576053.141489:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576053.141499:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576053.141502:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576053.141515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576053.141519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576053.141522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576053.141526:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576053.141528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576053.141531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576054.037542:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576054.037550:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576054.037556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576054.037557:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576054.037561:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576054.037563:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576054.037672:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576054.037679:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576054.037683:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576054.165514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576054.165533:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576054.165535:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576054.165546:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576054.165549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576054.165553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576054.165555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576054.165557:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576054.165560:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576055.061520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576055.061527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576055.061533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576055.061534:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576055.061538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576055.061539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576055.061646:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576055.061650:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576055.061653:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576055.189522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576055.189530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576055.189532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576055.189537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576055.189540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576055.189544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576055.189546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576055.189549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576055.189551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576056.086938:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576056.086946:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576056.086951:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576056.086952:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576056.086956:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576056.086957:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576056.087314:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576056.087318:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576056.087321:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576056.213522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576056.213532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576056.213534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576056.213539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576056.213542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576056.213547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576056.213549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576056.213551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576056.213554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576056.725592:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576056.725606:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788576056.725610:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576056.725615:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576056.725621:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576056.725629:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12987
00000400:00000200:2.0:1788576056.725640:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576056.725648:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576056.725652:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576056.725655:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576056.725658:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576056.725661:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576056.725668:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576056.725681:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576056.725684:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788576056.725687:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576056.725691:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788576056.725694:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788576056.725696:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788576056.725699:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576056.725701:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576056.725703:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576056.725705:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576056.725709:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788576056.725712:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576056.725714:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576056.725716:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576056.725720:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576056.725722:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576056.725724:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788576056.725726:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576056.725736:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03b80.
00000020:00000040:2.0:1788576056.725759:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788576056.725763:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576056.725765:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576056.725768:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576056.725771:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576056.725777:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576056.725794:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12c00.
00000100:00000001:2.0:1788576056.725803:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576056.725816:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576056.725828:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576056.725833:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576056.725838:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037d167ca] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576056.725861:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576056.725863:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576056.725866:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788576056.725867:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576056.725869:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576056.725871:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576056.725873:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12987
00000400:00000200:2.0:1788576056.725876:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576056.725879:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576056.725880:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576056.725882:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576056.725884:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576056.725886:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576056.725888:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576056.725890:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576056.725892:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788576056.725894:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576056.725897:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788576056.725899:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788576056.725902:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1788576056.725902:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788576056.725907:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576056.725908:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576056.725912:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1788576056.725912:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576056.725917:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576056.725922:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:0.0:1788576056.725923:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788576056.725927:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788576056.725935:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576056.725937:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576056.725941:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576056.725942:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576056.725945:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576056.725946:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576056.725949:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576056.725953:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576056.725957:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393255808:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1788576056.725961:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576056.725964:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576056.725964:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576056.725968:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576056.725972:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1788576056.725973:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:2.0:1788576056.725975:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576056.725978:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576056.725978:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576056.725983:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02d80.
02000000:00000001:0.0:1788576056.725984:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1788576056.725986:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788576056.725989:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576056.725991:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576056.725992:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:0.0:1788576056.725993:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576056.725996:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576056.726004:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576056.726009:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576056.726017:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
02000000:00000001:2.0:1788576056.726022:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576056.726024:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576056.726026:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393255808, portal 25
02000000:00000010:2.0:1788576056.726030:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f13400.
00000100:00000001:0.0:1788576056.726031:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576056.726036:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530786688 : -130484178764928 : ffff895348e03b80)
00000100:00000001:2.0:1788576056.726036:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576056.726043:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1788576056.726046:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576081 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576056.726055:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576056.726060:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.726066:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576056.726066:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009a9ccdb8] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788576056.726075:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788576056.726078:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576056.726080:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00080000:2.0:1788576056.726081:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:0.0:1788576056.726085:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393255808, offset 0
00000100:00080000:2.0:1788576056.726086:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788576056.726089:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576056.726092:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576056.726096:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:0.0:1788576056.726098:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788576056.726102:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12987
00000400:00000200:2.0:1788576056.726107:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576056.726111:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1788576056.726114:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1788576056.726117:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1788576056.726124:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1788576056.726125:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576056.726130:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1788576056.726132:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1788576056.726135:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576056.726138:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788576056.726139:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:2.0:1788576056.726143:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1788576056.726145:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000040:2.0:1788576056.726147:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:0.0:1788576056.726150:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:2.0:1788576056.726151:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:0.0:1788576056.726156:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331254:331254:256:4294967295] 192.168.202.110@tcp LPNI seq info [331211:331211:8:4294967295]
00000020:00000040:2.0:1788576056.726157:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:0.0:1788576056.726162:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1788576056.726162:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788576056.726167:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788576056.726175:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000200:0.0:1788576056.726176:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1788576056.726181:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1788576056.726186:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1788576056.726188:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000800:00000010:0.0:1788576056.726192:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000100:00000001:2.0:1788576056.726192:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576056.726202:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000800:00000200:0.0:1788576056.726207:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1788576056.726208:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:0.0:1788576056.726212:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1788576056.726214:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576056.726219:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1788576056.726222:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1788576056.726224:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.1:1788576056.726226:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
02000000:00000001:2.0:1788576056.726230:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1788576056.726233:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1788576056.726235:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000010:0.0:1788576056.726236:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000008:00100000:2.0:1788576056.726241:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:0.0:1788576056.726245:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00000001:2.0:1788576056.726245:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576056.726249:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576056.726252:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576056.726257:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576056.726258:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000010:2.0:1788576056.726261:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03100.
00000100:00000001:0.0:1788576056.726264:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000020:00000040:2.0:1788576056.726265:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788576056.726269:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:0.0:1788576056.726270:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576081 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1788576056.726270:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576056.726274:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576056.726276:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576056.726279:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576056.726285:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788576056.726291:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.726292:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00100000:2.0:1788576056.726298:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1788576056.726299:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576081 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726325:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576056.726326:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576056.726328:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576056.726331:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.726336:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576056.726339:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008692924c] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788576056.726342:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576056.726346:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307661241
00000100:00000010:0.0:1788576056.726354:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:2.0:1788576056.726355:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576056.726360:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788576056.726365:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:0.0:1788576056.726371:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:2.0:1788576056.726371:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12997)
00000100:00000001:0.0:1788576056.726382:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307661241
00000100:00000010:0.0:1788576056.726384:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7060.
00000100:00000001:0.0:1788576056.726387:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576056.726418:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576056.726421:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.726423:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576056.726427:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576056/real 1788576056]  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576056.726434:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576056.726435:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576056.726438:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576056.726440:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576056.726441:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576056.726443:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576056.726445:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576056.726451:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726455:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576056.726456:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726459:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576056.726460:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576056.726463:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726467:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576056.726470:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726474:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576056.726476:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726477:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576056.726478:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726484:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576056.726486:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576056.726489:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576056.726492:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576056.726494:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576056.726496:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576056.726501:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393255808:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576056.726505:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726506:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576056.726508:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03b80 x1875444393255808/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726512:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576056.726514:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576056.726516:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576056.726518:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576056.726520:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576056.726522:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576056.726523:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576056.726525:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12c00.
02000000:00000001:0.0:1788576056.726528:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576056.726530:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576056.726531:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03b80.
00000100:00000001:0.0:1788576056.726534:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576056.726535:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726536:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726543:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576056.726544:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.726546:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576056.726547:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576056.726548:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576056.726550:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726555:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576056.726556:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576056.726557:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576056.726559:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576056.726560:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576056.726561:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255936:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576056.726563:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576056.726564:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576056.726565:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576056.726566:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576056.726568:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576056.726569:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576056.726572:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576056.726574:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576056.726576:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576056.726578:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393255936, portal 10
00000100:00000001:0.0:1788576056.726579:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576056.726580:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530783104 : -130484178768512 : ffff895348e02d80)
00000100:00000040:0.0:1788576056.726585:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576081 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726590:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576056.726591:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576056.726593:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576056.726594:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393255936, offset 0
00000400:00000200:0.0:1788576056.726597:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576056.726599:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576056.726601:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576056.726603:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576056.726605:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576056.726606:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576056.726607:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576056.726609:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331255:331255:256:4294967295] 192.168.202.110@tcp LPNI seq info [331212:331212:8:4294967295]
00000400:00000200:0.0:1788576056.726612:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576056.726617:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576056.726619:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576056.726621:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576056.726624:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576056.726626:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788576056.726634:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576056.726636:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788576056.726638:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576056.726641:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576056.726643:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576056.726647:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576056.726650:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576056.726651:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576056.726653:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576056.726657:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576081 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726662:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576056.726665:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576081 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726674:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.726675:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576056.726677:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576056.726680:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576056.726683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576056.726684:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307660327
00000100:00000001:0.0:1788576056.726686:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307661241
00000100:00000001:0.0:1788576056.726688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307660327
00000100:00000001:0.0:1788576056.726689:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307661241
00000100:00000001:0.0:1788576056.726690:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576056.726691:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.726693:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576056.726694:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576056/real 1788576056]  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576056.726699:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576056.726700:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576056.726701:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576056.726702:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576056.726703:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576056.726704:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576056.726706:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576056.726711:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726715:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576056.726716:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726717:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576056.726717:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576056.726719:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726724:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576056.726726:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726736:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576056.726747:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726749:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576056.726750:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726752:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576056.726754:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576056.726757:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576056.726760:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576056.726762:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576056.726766:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576056.726775:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393255936:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576056.726779:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576056.726782:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726789:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576056.726790:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576056.726793:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576056.726794:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576056.726796:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576056.726799:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576056.726801:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576056.726803:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576056.726805:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576056.726806:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576056.726808:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788576056.726810:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576056.726813:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576056.726814:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576056.726817:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576056.726819:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393256064, portal 4
00000100:00000001:0.0:1788576056.726820:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576056.726821:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784000 : -130484178767616 : ffff895348e03100)
00000100:00000040:0.0:1788576056.726825:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576081 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726830:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576056.726831:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576056.726832:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576056.726833:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393256064, offset 0
00000400:00000200:0.0:1788576056.726836:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576056.726838:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576056.726840:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576056.726842:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576056.726843:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576056.726845:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576056.726847:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576056.726852:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331256:331256:256:4294967295] 192.168.202.110@tcp LPNI seq info [331213:331213:8:4294967295]
00000400:00000200:0.0:1788576056.726855:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576056.726859:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576056.726863:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576056.726866:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576056.726869:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576056.726871:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1788576056.726874:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576056.726876:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1788576056.726878:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576056.726880:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576056.726882:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576056.726884:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576056.726887:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576056.726888:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576056.726889:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576056.726891:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576081 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726896:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576056.726898:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576081 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726903:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.726905:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576056.726907:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576056.726910:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576056.726913:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576056.726915:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307660473
00000100:00000001:0.0:1788576056.726917:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307661241
00000100:00000001:0.0:1788576056.726919:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307660473
00000100:00000001:0.0:1788576056.726921:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307661241
00000100:00000001:0.0:1788576056.726924:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576056.726926:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.726927:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576056.726931:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576056/real 1788576056]  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576056.726939:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576056.726940:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576056.726942:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576056.726944:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576056.726946:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576056.726949:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576056.726952:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576056.726960:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726967:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576056.726968:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726970:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576056.726972:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576056.726975:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726982:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576056.726986:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.726993:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576056.726995:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726997:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576056.726998:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.726999:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576056.727001:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576056.727004:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576056.727007:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576056.727009:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576056.727012:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576056.727019:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576056.727023:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.727024:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576056.727028:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02d80 x1875444393255936/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.727034:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576056.727036:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576056.727037:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5c00.
02000000:00000001:0.0:1788576056.727040:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576056.727041:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576056.727043:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576056.727045:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576056.727046:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f13400.
02000000:00000001:0.0:1788576056.727049:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576056.727050:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576056.727051:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02d80.
00000100:00000001:0.0:1788576056.727054:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576056.727055:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.727057:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576056.727060:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03100 x1875444393256064/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576081 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576056.727066:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576056.727067:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576056.727068:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6800.
02000000:00000001:0.0:1788576056.727070:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576056.727071:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576056.727073:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576056.727075:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576056.727076:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788576056.727077:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576056.727078:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576056.727079:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03100.
00000100:00000001:0.0:1788576056.727081:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576056.727082:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.727084:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576056.727087:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576056.727088:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576056.727090:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576056.727091:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576057.109530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576057.109536:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576057.109541:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576057.109544:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576057.109546:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576057.109548:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576057.109550:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576057.109551:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576057.109553:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576057.109554:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576057.109556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576057.109557:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576057.109558:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576057.109559:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576057.109559:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576057.109560:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576057.109561:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576057.109562:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576057.109610:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576057.109614:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576057.109617:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576057.237522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576057.237530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576057.237533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576057.237538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576057.237542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576057.237545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576057.237548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576057.237551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576057.237554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576057.685494:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307661241, at 4307661248
00000100:00000040:1.0:1788576057.685494:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307661241, at 4307661248
00000100:00000010:0.0:1788576057.685507:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:1.0:1788576057.685510:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7060.
00000800:00000200:0.0:1788576057.749576:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788576057.749585:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788576057.749590:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788576057.749753:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1788576057.749769:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788576057.749770:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788576057.749777:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788576057.749780:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788576058.133510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576058.133518:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576058.133523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576058.133525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576058.133528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576058.133529:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576058.133582:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576058.133585:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576058.133588:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576058.261544:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576058.261550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576058.261552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576058.261556:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576058.261559:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576058.261561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576058.261563:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576058.261565:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576058.261568:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576059.157563:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576059.157573:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576059.157575:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576059.157579:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576059.157584:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576059.157587:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576059.157589:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576059.157592:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576059.157596:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576059.157600:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576059.157604:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576059.157606:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576059.157671:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576059.157675:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576059.157678:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576059.285516:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576059.285525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576059.285527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576059.285532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576059.285536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576059.285540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576059.285543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576059.285545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576059.285548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576060.181513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576060.181526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576060.181535:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576060.181537:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576060.181540:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576060.181541:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576060.181641:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576060.181644:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576060.181646:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576060.309499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576060.309507:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576060.309509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576060.309515:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576060.309519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576060.309522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576060.309525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576060.309528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576060.309531:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576061.206986:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576061.206993:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576061.206999:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576061.207001:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576061.207004:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576061.207005:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576061.207077:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576061.207081:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576061.207084:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576061.333515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576061.333523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576061.333525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576061.333531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576061.333535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576061.333539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576061.333541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576061.333543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576061.333546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576061.845485:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576061.845496:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788576061.845498:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576061.845501:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576061.845505:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576061.845511:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12992
00000400:00000200:2.0:1788576061.845520:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576061.845526:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576061.845529:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576061.845532:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845535:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576061.845537:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845539:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576061.845547:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576061.845549:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788576061.845552:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576061.845555:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788576061.845557:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788576061.845559:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788576061.845562:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845564:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576061.845566:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845567:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576061.845570:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788576061.845573:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845575:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576061.845578:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576061.845583:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576061.845585:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576061.845588:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788576061.845590:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576061.845599:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00e00.
00000020:00000040:2.0:1788576061.845602:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788576061.845606:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576061.845607:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576061.845610:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576061.845612:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576061.845615:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576061.845621:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788576061.845626:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576061.845633:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576061.845641:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576061.845644:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576061.845647:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5f665f0] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576061.845664:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576061.845667:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576061.845670:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788576061.845672:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576061.845673:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576061.845676:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576061.845678:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12992
00000400:00000200:2.0:1788576061.845682:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576061.845685:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576061.845687:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1788576061.845690:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1788576061.845690:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845692:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788576061.845694:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1788576061.845694:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845696:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788576061.845697:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1788576061.845698:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576061.845699:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788576061.845701:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:0.0:1788576061.845703:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1788576061.845703:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788576061.845705:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788576061.845707:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788576061.845710:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576061.845711:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576061.845712:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788576061.845713:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576061.845714:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845715:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000001:0.0:1788576061.845717:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1788576061.845717:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:0.0:1788576061.845719:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1788576061.845720:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1788576061.845721:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576061.845722:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576061.845724:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1788576061.845725:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393256192:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576061.845728:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576061.845729:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576061.845730:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576061.845732:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1788576061.845732:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:0.0:1788576061.845734:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576061.845736:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:2.0:1788576061.845736:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788576061.845738:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576061.845740:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576061.845741:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
00000100:00000010:2.0:1788576061.845742:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01c00.
02000000:00000001:0.0:1788576061.845743:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1788576061.845744:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788576061.845746:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1788576061.845748:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788576061.845748:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576061.845759:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576061.845761:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1788576061.845762:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
02000000:00000001:2.0:1788576061.845763:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576061.845767:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f12000.
00000400:00000010:0.0:1788576061.845768:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000001:2.0:1788576061.845770:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576061.845774:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393256192, portal 25
00000100:00100000:2.0:1788576061.845774:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576061.845776:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576061.845778:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775040 : -130484178776576 : ffff895348e00e00)
00000100:00000001:2.0:1788576061.845782:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1788576061.845783:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576086 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576061.845784:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576061.845788:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000de3acaa1] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.845790:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576061.845793:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576061.845795:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000001:2.0:1788576061.845796:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576061.845799:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393256192, offset 0
00000100:00080000:2.0:1788576061.845799:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576061.845802:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788576061.845803:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1788576061.845804:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1788576061.845805:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576061.845807:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576061.845809:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12992
00000400:00000200:0.0:1788576061.845812:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576061.845812:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576061.845815:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576061.845817:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1788576061.845818:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1788576061.845819:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845821:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576061.845823:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000200:0.0:1788576061.845824:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1788576061.845825:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000200:0.0:1788576061.845827:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576061.845830:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576061.845833:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:2.0:1788576061.845836:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:0.0:1788576061.845837:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331257:331257:256:4294967295] 192.168.202.110@tcp LPNI seq info [331214:331214:8:4294967295]
00000020:00000040:2.0:1788576061.845838:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788576061.845840:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576061.845842:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788576061.845843:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788576061.845846:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:0.0:1788576061.845848:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1788576061.845848:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845850:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576061.845852:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845854:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576061.845856:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000200:0.0:1788576061.845857:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1788576061.845859:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576061.845861:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788576061.845863:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1788576061.845863:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576061.845865:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576061.845867:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788576061.845868:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000100:00000001:2.0:1788576061.845869:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576061.845871:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:0.0:1788576061.845873:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000008:00000001:2.0:1788576061.845873:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576061.845875:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788576061.845877:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000010:2.0:1788576061.845878:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00a80.
00000020:00000040:2.0:1788576061.845880:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788576061.845882:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:0.1:1788576061.845882:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1788576061.845883:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576061.845885:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:0.1:1788576061.845885:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
02000000:00000001:2.0:1788576061.845886:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576061.845888:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1788576061.845890:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000010:2.0:1788576061.845891:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11400.
00000800:00000010:0.0:1788576061.845892:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000100:00000001:2.0:1788576061.845894:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576061.845899:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576061.845903:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00100000:2.0:1788576061.845906:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788576061.845908:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576061.845911:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:2.0:1788576061.845912:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576061.845914:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.845915:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:2.0:1788576061.845918:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000389a8694] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576061.845919:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576086 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576061.845926:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.845928:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576061.845928:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788576061.845931:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576086 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788576061.845932:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788576061.845936:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13003)
00000100:00000001:0.0:1788576061.845937:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.845948:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576061.845953:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576061.845957:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576061.845959:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307666361
00000100:00000010:0.0:1788576061.845964:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576061.845969:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576061.845974:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307666361
00000100:00000010:0.0:1788576061.845977:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7f60.
00000100:00000001:0.0:1788576061.845980:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576061.845982:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576061.845985:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.845988:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576061.845992:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576061/real 1788576061]  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576061.846001:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576061.846002:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576061.846004:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576061.846007:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576061.846009:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576061.846011:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576061.846014:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576061.846022:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846028:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576061.846030:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846032:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576061.846034:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576061.846038:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846045:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576061.846050:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846057:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576061.846060:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846061:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576061.846062:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846064:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576061.846066:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576061.846069:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576061.846073:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576061.846075:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576061.846079:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576061.846088:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393256192:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576061.846093:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846094:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576061.846097:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00e00 x1875444393256192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846103:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576061.846106:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576061.846108:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788576061.846111:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576061.846113:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576061.846115:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576061.846117:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576061.846120:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788576061.846123:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576061.846124:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576061.846126:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00e00.
00000100:00000001:0.0:1788576061.846129:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576061.846130:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846132:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846137:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576061.846138:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.846139:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576061.846140:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576061.846141:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576061.846144:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846148:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576061.846149:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576061.846151:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576061.846152:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576061.846153:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576061.846155:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576061.846157:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576061.846158:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576061.846159:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576061.846160:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576061.846162:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788576061.846163:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576061.846166:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576061.846168:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576061.846170:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576061.846172:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393256320, portal 10
00000100:00000001:0.0:1788576061.846173:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576061.846174:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530778624 : -130484178772992 : ffff895348e01c00)
00000100:00000040:0.0:1788576061.846179:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576086 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846183:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576061.846184:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576061.846185:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576061.846187:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393256320, offset 0
00000400:00000200:0.0:1788576061.846189:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576061.846191:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576061.846194:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576061.846196:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576061.846197:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576061.846199:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576061.846200:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576061.846202:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331258:331258:256:4294967295] 192.168.202.110@tcp LPNI seq info [331215:331215:8:4294967295]
00000400:00000200:0.0:1788576061.846205:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576061.846209:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576061.846212:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576061.846214:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576061.846217:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576061.846219:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788576061.846221:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576061.846223:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1788576061.846226:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576061.846227:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576061.846229:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576061.846231:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576061.846233:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576061.846234:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576061.846236:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576061.846238:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576086 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846251:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576061.846254:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576086 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846266:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.846267:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576061.846270:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576061.846274:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576061.846277:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576061.846279:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307665446
00000100:00000001:0.0:1788576061.846281:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307666361
00000100:00000001:0.0:1788576061.846283:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307665446
00000100:00000001:0.0:1788576061.846285:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307666361
00000100:00000001:0.0:1788576061.846288:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576061.846290:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.846291:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576061.846295:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576061/real 1788576061]  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576061.846303:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576061.846303:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576061.846305:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576061.846306:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576061.846307:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576061.846309:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576061.846311:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576061.846316:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846320:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576061.846321:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846322:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576061.846324:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576061.846327:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846334:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576061.846338:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846344:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576061.846346:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846348:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576061.846349:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846351:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576061.846365:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576061.846368:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576061.846371:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576061.846373:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576061.846377:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576061.846385:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576061.846388:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576061.846390:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846410:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576061.846410:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576061.846412:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576061.846413:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576061.846414:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576061.846416:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256448:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576061.846418:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576061.846419:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576061.846421:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576061.846422:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576061.846425:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788576061.846426:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576061.846429:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576061.846430:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576061.846433:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576061.846435:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393256448, portal 4
00000100:00000001:0.0:1788576061.846436:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576061.846437:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530774144 : -130484178777472 : ffff895348e00a80)
00000100:00000040:0.0:1788576061.846440:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576086 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846444:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576061.846445:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576061.846446:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576061.846448:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393256448, offset 0
00000400:00000200:0.0:1788576061.846450:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576061.846452:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576061.846454:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576061.846456:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576061.846458:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576061.846459:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576061.846460:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576061.846462:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331259:331259:256:4294967295] 192.168.202.110@tcp LPNI seq info [331216:331216:8:4294967295]
00000400:00000200:0.0:1788576061.846473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576061.846485:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576061.846490:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576061.846493:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576061.846496:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576061.846499:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1788576061.846503:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576061.846505:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:0.0:1788576061.846509:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576061.846511:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576061.846514:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576061.846518:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576061.846521:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576061.846522:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576061.846524:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576061.846527:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576086 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846535:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576061.846538:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576086 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846544:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.846546:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576061.846549:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576061.846551:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576061.846555:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576061.846557:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307665593
00000100:00000001:0.0:1788576061.846559:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307666361
00000100:00000001:0.0:1788576061.846561:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307665593
00000100:00000001:0.0:1788576061.846563:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307666361
00000100:00000001:0.0:1788576061.846564:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576061.846566:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.846567:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576061.846569:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576061/real 1788576061]  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576061.846574:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576061.846575:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576061.846576:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576061.846577:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576061.846578:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576061.846580:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576061.846582:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576061.846587:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846590:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576061.846591:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846592:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576061.846592:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576061.846594:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846598:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576061.846600:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846604:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576061.846605:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846606:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576061.846607:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846608:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576061.846609:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576061.846610:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576061.846612:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576061.846613:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576061.846615:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576061.846619:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256448:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576061.846621:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846623:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576061.846625:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01c00 x1875444393256320/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846629:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576061.846630:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576061.846631:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd6000.
02000000:00000001:0.0:1788576061.846633:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576061.846634:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576061.846634:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576061.846636:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576061.846637:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f12000.
02000000:00000001:0.0:1788576061.846638:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576061.846639:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576061.846640:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01c00.
00000100:00000001:0.0:1788576061.846642:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576061.846642:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846644:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576061.846645:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00a80 x1875444393256448/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576086 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576061.846648:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576061.846649:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576061.846650:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd7400.
02000000:00000001:0.0:1788576061.846651:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576061.846651:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576061.846652:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576061.846653:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576061.846654:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11400.
02000000:00000001:0.0:1788576061.846655:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576061.846655:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576061.846656:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00a80.
00000100:00000001:0.0:1788576061.846657:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576061.846658:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846659:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576061.846661:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576061.846662:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576061.846663:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576061.846664:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576062.229577:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576062.229586:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576062.229592:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576062.229594:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576062.229597:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576062.229598:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576062.229697:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576062.229704:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576062.229709:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576062.357565:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576062.357577:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576062.357580:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576062.357588:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576062.357592:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576062.357597:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576062.357600:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576062.357602:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576062.357605:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576062.805596:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307666361, at 4307666368
00000100:00000010:0.0:1788576062.805617:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000040:1.0:1788576062.805829:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307666361, at 4307666368
00000100:00000010:1.0:1788576062.805837:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7f60.
00010000:00000001:3.0:1788576063.253550:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576063.253559:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576063.253567:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576063.253568:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576063.253571:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576063.253573:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576063.253641:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576063.253646:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576063.253650:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576063.381547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576063.381556:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576063.381558:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576063.381564:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576063.381567:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576063.381571:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576063.381574:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576063.381576:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576063.381579:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576064.277631:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576064.277642:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576064.277661:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576064.277663:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576064.277667:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576064.277669:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576064.277853:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576064.277858:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576064.277862:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576064.408351:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576064.408361:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576064.408364:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576064.408371:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576064.408376:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576064.408381:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576064.408384:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576064.408387:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576064.408408:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576065.301532:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576065.301540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576065.301547:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576065.301548:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576065.301553:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576065.301554:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576065.301598:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576065.301603:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576065.301607:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576065.429547:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576065.429556:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576065.429558:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576065.429566:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576065.429570:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576065.429576:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576065.429579:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576065.429582:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576065.429585:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576066.325513:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576066.325519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576066.325525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576066.325526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576066.325530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576066.325531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576066.327030:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576066.327037:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576066.327042:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576066.453549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576066.453559:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576066.453562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576066.453569:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576066.453574:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576066.453578:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576066.453581:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576066.453584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576066.453587:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576066.965738:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576066.965753:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788576066.965756:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576066.965762:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576066.965780:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576066.965790:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 12998
00000400:00000200:2.0:1788576066.965801:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576066.965808:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576066.965812:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576066.965816:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.965820:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576066.965823:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.965825:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576066.965829:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576066.965832:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788576066.965835:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576066.965838:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788576066.965841:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788576066.965843:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788576066.965847:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.965849:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576066.965851:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.965853:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576066.965857:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788576066.965861:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576066.965864:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576066.965868:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576066.965871:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576066.965887:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576066.965889:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788576066.965892:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576066.965904:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03800.
00000020:00000040:2.0:1788576066.965910:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788576066.965914:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576066.965916:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576066.965919:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576066.965923:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576066.965928:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576066.965937:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000100:00000001:2.0:1788576066.965946:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576066.965958:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576066.965970:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576066.965976:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576066.965981:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014d98894] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.966040:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576066.966046:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576066.966049:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576066.966057:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.966066:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576066.966069:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576066.966073:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576066.966077:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576066.966088:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576066.966092:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393256576:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576066.966097:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576066.966103:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576066.966106:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576066.966109:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576066.966116:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788576066.966118:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576066.966124:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576066.966130:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576066.966138:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576066.966147:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393256576, portal 25
00000100:00000001:0.0:1788576066.966150:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576066.966152:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530785792 : -130484178765824 : ffff895348e03800)
00000100:00000040:0.0:1788576066.966159:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576091 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.966173:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576066.966176:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576066.966180:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576066.966183:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393256576, offset 0
00000400:00000200:0.0:1788576066.966190:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576066.966201:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576066.966209:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576066.966215:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576066.966219:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576066.966223:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576066.966227:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576066.966231:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331260:331260:256:4294967295] 192.168.202.110@tcp LPNI seq info [331217:331217:8:4294967295]
00000400:00000200:0.0:1788576066.966236:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576066.966247:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576066.966254:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576066.966260:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576066.966268:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576066.966273:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788576066.966279:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576066.966283:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788576066.966287:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576066.966290:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576066.966298:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576066.966303:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576066.966310:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576066.966312:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576066.966318:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576066.966322:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576091 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576066.966350:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576066.966354:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576066.966358:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788576066.966359:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576066.966361:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576066.966364:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576066.966367:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12998
00000400:00000200:2.0:1788576066.966371:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576066.966373:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576066.966376:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576066.966379:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.966381:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576066.966383:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.966385:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576066.966387:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576066.966388:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788576066.966390:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576066.966414:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788576066.966417:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788576066.966419:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788576066.966423:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.966425:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576066.966428:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.966429:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576066.966432:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788576066.966435:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576066.966438:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576066.966440:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576066.966447:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576066.966449:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576066.966451:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576066.966456:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788576066.966460:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576066.966462:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576066.966466:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02a00.
00000020:00000040:2.0:1788576066.966468:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788576066.966471:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576066.966472:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576066.966474:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576066.966476:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576066.966477:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576066.966481:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000001:2.0:1788576066.966486:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576066.966491:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576066.966499:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576066.966502:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576066.966505:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cce910e4] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.966828:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576066.966834:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576091 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.966849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576066.966851:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576066.966859:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576066.966864:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576066.966868:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307671482
00000100:00000010:0.0:1788576066.967640:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576066.967650:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576066.967660:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307671482
00000100:00000010:0.0:1788576066.967664:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7240.
00000100:00000001:0.0:1788576066.967668:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576066.967671:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576066.967675:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576066.967677:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576066.967754:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576066/real 1788576066]  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576066.967780:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576066.967782:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576066.967785:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576066.967789:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576066.967792:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576066.967795:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576066.967800:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576066.967809:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.967816:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576066.967818:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.967821:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576066.967823:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576066.967827:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.967835:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576066.967840:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.967846:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576066.967850:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.967851:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576066.967852:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.967855:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576066.967858:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576066.967861:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576066.967866:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576066.967868:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576066.967873:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576066.967940:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393256576:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576066.967946:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.967948:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576066.967951:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03800 x1875444393256576/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.967957:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576066.967960:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576066.967963:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788576066.967966:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576066.967968:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576066.967969:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576066.967971:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576066.967974:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788576066.967978:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576066.967980:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576066.967982:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03800.
00000100:00000001:0.0:1788576066.967986:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576066.967987:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.967989:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.968063:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576066.968066:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576066.968069:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576066.968071:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576066.968073:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576066.968078:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.968085:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576066.968087:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576066.968089:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576066.968092:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576066.968094:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576066.968097:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256704:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576066.968100:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576066.968102:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576066.968104:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576066.968106:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576066.968110:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788576066.968112:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576066.968118:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576066.968121:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576066.968126:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576066.968130:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393256704, portal 10
00000100:00000001:0.0:1788576066.968131:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576066.968133:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530782208 : -130484178769408 : ffff895348e02a00)
00000100:00000040:0.0:1788576066.968140:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576091 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.968147:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576066.968149:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576066.968152:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576066.968155:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393256704, offset 0
00000400:00000200:0.0:1788576066.968160:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576066.968166:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576066.968173:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576066.968177:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576066.968180:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576066.968183:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576066.968187:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576066.968190:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331261:331261:256:4294967295] 192.168.202.110@tcp LPNI seq info [331218:331218:8:4294967295]
00000400:00000200:0.0:1788576066.968194:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576066.968203:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576066.968209:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576066.968213:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000100:00000001:2.0:1788576066.968328:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576066.968334:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576066.968338:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788576066.968340:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576066.968343:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576066.968347:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576066.968350:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 12998
00000400:00000200:2.0:1788576066.968355:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576066.968358:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576066.968361:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576066.968364:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.968366:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576066.968368:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.968370:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576066.968373:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576066.968374:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788576066.968377:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576066.968379:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788576066.968381:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788576066.968383:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788576066.968385:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.968387:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576066.968388:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576066.968405:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576066.968408:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788576066.968412:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576066.968415:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576066.968417:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576066.968422:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576066.968424:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576066.968426:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576066.968430:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788576066.968433:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576066.968434:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576066.968439:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01f80.
00000020:00000040:2.0:1788576066.968443:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788576066.968446:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576066.968448:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576066.968451:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576066.968453:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576066.968456:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576066.968463:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10000.
00000100:00000001:2.0:1788576066.968469:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576066.968475:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576066.968483:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576066.968487:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576066.968491:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f8c8ec7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576066.968500:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576066.968503:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788576066.968507:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788576066.968512:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13008)
00000800:00000200:0.0:1788576066.968571:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576066.968576:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788576066.968581:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576066.968584:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788576066.968588:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576066.968591:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576066.968596:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576066.968600:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576066.968606:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576066.968608:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576066.968611:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576066.968615:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576091 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.968622:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576066.968626:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576091 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.968633:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576066.968635:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576066.968641:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576066.968645:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576066.968650:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576066.968652:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307670569
00000100:00000001:0.0:1788576066.968655:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307671482
00000100:00000001:0.0:1788576066.968657:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307670569
00000100:00000001:0.0:1788576066.968660:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307671482
00000100:00000001:0.0:1788576066.968662:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576066.968664:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576066.968666:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576066.968670:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576066/real 1788576066]  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576066.968677:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576066.968678:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576066.968680:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576066.968739:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576066.968743:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576066.968746:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576066.968750:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576066.968761:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.968779:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576066.968781:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.968783:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576066.968784:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576066.968788:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.968794:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576066.968798:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.968805:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576066.968807:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.968809:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576066.968810:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.968811:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576066.968814:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576066.968816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576066.968819:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576066.968821:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576066.968825:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576066.968831:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256704:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576066.968835:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.968837:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576066.968840:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02a00 x1875444393256704/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.968846:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576066.968847:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576066.968849:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788576066.968851:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576066.968853:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576066.968854:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576066.968856:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576066.968857:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:0.0:1788576066.968859:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576066.968860:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576066.968862:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02a00.
00000100:00000001:0.0:1788576066.968864:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576066.968865:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.968868:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.968871:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576066.968936:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576066.968938:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576066.968940:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576066.968941:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576066.968946:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.968952:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576066.968953:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576066.968955:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576066.968956:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576066.968958:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576066.968961:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256832:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576066.968965:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576066.968966:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576066.968968:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576066.968969:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576066.968973:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788576066.968974:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576066.968978:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576066.968980:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576066.968984:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576066.968987:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393256832, portal 4
00000100:00000001:0.0:1788576066.968989:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576066.968990:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530779520 : -130484178772096 : ffff895348e01f80)
00000100:00000040:0.0:1788576066.969058:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576091 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.969065:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576066.969068:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576066.969071:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576066.969074:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393256832, offset 0
00000400:00000200:0.0:1788576066.969078:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576066.969082:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576066.969086:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576066.969089:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576066.969091:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576066.969094:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576066.969097:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576066.969099:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331262:331262:256:4294967295] 192.168.202.110@tcp LPNI seq info [331219:331219:8:4294967295]
00000400:00000200:0.0:1788576066.969103:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576066.969109:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576066.969113:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576066.969116:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576066.969120:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576066.969122:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1788576066.969126:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576066.969129:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1788576066.969132:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576066.969134:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576066.969137:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576066.969140:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576066.969143:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576066.969145:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576066.969148:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576066.969151:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576091 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.969159:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576066.969162:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576091 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.969168:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576066.969170:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576066.969173:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576066.969176:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576066.969180:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576066.969182:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307670715
00000100:00000001:0.0:1788576066.969185:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307671482
00000100:00000001:0.0:1788576066.969187:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307670715
00000100:00000001:0.0:1788576066.969189:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307671482
00000100:00000001:0.0:1788576066.969191:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576066.969193:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576066.969194:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576066.969198:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576066/real 1788576066]  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576066.969204:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576066.969205:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576066.969207:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576066.969209:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576066.969211:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576066.969214:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576066.969216:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576066.969224:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.969229:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576066.969231:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.969232:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576066.969234:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576066.969237:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.969242:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576066.969246:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.969252:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576066.969254:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.969256:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576066.969257:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.969259:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576066.969261:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576066.969263:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576066.969266:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576066.969268:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576066.969272:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576066.969278:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393256832:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576066.969282:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.969283:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576066.969286:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01f80 x1875444393256832/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576091 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576066.969292:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576066.969293:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576066.969295:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4400.
02000000:00000001:0.0:1788576066.969297:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576066.969299:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576066.969300:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576066.969302:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576066.969304:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10000.
02000000:00000001:0.0:1788576066.969306:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576066.969307:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576066.969308:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01f80.
00000100:00000001:0.0:1788576066.969311:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576066.969312:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.969314:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576066.969317:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576066.969319:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576066.969320:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576066.969322:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576067.349526:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576067.349533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576067.349539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576067.349542:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576067.349544:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576067.349547:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576067.349551:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576067.349553:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576067.349554:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576067.349557:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576067.349561:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576067.349562:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576067.349564:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576067.349565:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576067.349567:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576067.349568:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576067.349570:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576067.349572:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576067.349855:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576067.349860:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576067.349863:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576067.480710:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576067.480717:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576067.480720:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576067.480726:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576067.480731:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576067.480735:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576067.480748:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576067.480752:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576067.480755:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576067.925546:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307671482, at 4307671488
00000100:00000040:1.0:1788576067.925546:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307671482, at 4307671488
00000100:00000010:0.0:1788576067.925567:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000010:1.0:1788576067.925571:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7240.
00010000:00000001:3.0:1788576068.373663:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576068.373669:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576068.373675:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576068.373677:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576068.373680:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576068.373681:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576068.374018:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576068.374020:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576068.374023:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576068.501518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576068.501524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576068.501525:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576068.501529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576068.501532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576068.501535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576068.501537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576068.501539:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576068.501541:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576069.397578:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576069.397589:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576069.397592:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576069.397596:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576069.397603:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576069.397606:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576069.397608:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576069.397612:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576069.397616:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576069.397617:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576069.397622:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576069.397623:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576069.397818:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576069.397822:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576069.397825:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576069.525613:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576069.525625:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576069.525628:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576069.525636:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576069.525641:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576069.525647:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576069.525650:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576069.525654:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576069.525657:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576070.421608:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576070.421619:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576070.421627:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576070.421628:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576070.421633:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576070.421634:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576070.421713:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576070.421718:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576070.421721:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576070.549563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576070.549572:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576070.549574:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576070.549580:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576070.549585:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576070.549594:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576070.549598:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576070.549601:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576070.549605:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576071.445548:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576071.445554:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576071.445560:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576071.445562:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576071.445565:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576071.445566:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576071.448857:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576071.448864:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576071.448869:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576071.573563:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576071.573573:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576071.573575:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576071.573582:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576071.573586:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576071.573592:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576071.573596:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576071.573599:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576071.573603:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576072.085659:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576072.085669:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788576072.085672:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576072.085676:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576072.085680:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576072.085687:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13003
00000400:00000200:0.0:1788576072.085696:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576072.085702:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576072.085705:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576072.085708:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085711:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576072.085713:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085716:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576072.085720:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576072.085722:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788576072.085724:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576072.085727:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788576072.085729:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788576072.085731:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788576072.085733:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085736:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576072.085738:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085739:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576072.085742:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788576072.085745:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085747:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576072.085749:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576072.085752:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576072.085755:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576072.085757:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576072.085759:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576072.085768:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788576072.085771:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788576072.085775:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576072.085788:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576072.085792:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576072.085794:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576072.085798:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576072.085807:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788576072.085814:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576072.085824:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576072.085834:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576072.085838:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576072.085841:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576072.085866:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576072.085870:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576072.085873:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788576072.085875:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576072.085876:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576072.085879:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576072.085882:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13003
00000400:00000200:0.0:1788576072.085886:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576072.085889:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576072.085892:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576072.085894:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085896:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576072.085898:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085900:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576072.085902:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576072.085903:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788576072.085905:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576072.085907:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788576072.085909:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788576072.085911:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788576072.085913:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085915:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576072.085916:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085919:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576072.085921:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788576072.085923:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576072.085926:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576072.085928:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576072.085933:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576072.085935:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576072.085937:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576072.085941:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788576072.085944:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576072.085945:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576072.085949:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788576072.085951:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788576072.085954:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576072.085956:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576072.085958:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576072.085959:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576072.085961:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576072.085967:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788576072.085970:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576072.085975:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576072.085982:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576072.085984:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576072.085987:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576072.085994:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576072.085996:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576072.085998:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576072.086000:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576072.086001:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576072.086003:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576072.086006:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13003
00000400:00000200:0.0:1788576072.086009:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576072.086011:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576072.086013:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576072.086015:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.086017:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576072.086019:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.086021:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576072.086023:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576072.086024:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788576072.086025:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576072.086027:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788576072.086029:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788576072.086031:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788576072.086034:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.086036:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576072.086037:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576072.086039:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576072.086041:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788576072.086044:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576072.086046:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576072.086047:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576072.086050:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576072.086051:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576072.086053:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576072.086055:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788576072.086057:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576072.086059:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576072.086061:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000020:00000040:0.0:1788576072.086064:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788576072.086066:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576072.086067:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576072.086068:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576072.086070:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576072.086071:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576072.086074:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788576072.086077:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576072.086080:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576072.086087:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576072.086088:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576072.086091:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576072.086098:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576072.086101:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788576072.086105:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788576072.086109:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13013)
00000100:00000001:2.0:1788576072.086161:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576072.086165:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576072.086167:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576072.086174:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086183:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576072.086186:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576072.086189:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576072.086191:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576072.086193:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576072.086196:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393256960:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576072.086200:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576072.086204:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576072.086206:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576072.086208:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576072.086213:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788576072.086215:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576072.086220:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576072.086223:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576072.086229:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576072.086235:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393256960, portal 25
00000100:00000001:2.0:1788576072.086237:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576072.086239:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000040:2.0:1788576072.086244:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086250:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576072.086253:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576072.086255:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576072.086259:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393256960, offset 0
00000400:00000200:2.0:1788576072.086265:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576072.086273:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576072.086279:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576072.086284:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576072.086288:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576072.086291:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576072.086294:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576072.086297:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331263:331263:256:4294967295] 192.168.202.110@tcp LPNI seq info [331220:331220:8:4294967295]
00000400:00000200:2.0:1788576072.086302:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576072.086311:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576072.086318:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576072.086323:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576072.086329:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576072.086333:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788576072.086338:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576072.086341:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788576072.086345:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576072.086347:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576072.086353:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576072.086358:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576072.086362:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576072.086365:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576072.086369:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576072.086372:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086381:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576072.086384:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576097 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086407:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576072.086409:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576072.086416:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576072.086420:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576072.086423:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307676602
00000100:00000010:2.0:1788576072.086427:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cae0.
00000100:00000001:2.0:1788576072.086434:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576072.086438:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307676602
00000100:00000010:2.0:1788576072.086441:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c660.
00000100:00000001:2.0:1788576072.086443:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576072.086446:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576072.086448:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576072.086451:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576072.086455:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576072/real 1788576072]  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576072.086468:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576072.086470:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576072.086472:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576072.086474:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576072.086475:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576072.086478:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576072.086480:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576072.086488:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086493:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576072.086495:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.086497:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576072.086499:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576072.086503:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086509:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576072.086513:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086519:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576072.086522:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.086524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576072.086525:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.086527:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576072.086529:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576072.086532:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576072.086535:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576072.086538:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576072.086542:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576072.086550:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393256960:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576072.086554:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576072.086558:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086564:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576072.086565:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576072.086567:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576072.086569:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576072.086571:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576072.086574:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576072.086577:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576072.086579:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576072.086580:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576072.086582:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576072.086585:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576072.086587:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576072.086590:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576072.086593:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576072.086596:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576072.086599:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393257088, portal 10
00000100:00000001:2.0:1788576072.086601:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576072.086602:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00000040:2.0:1788576072.086609:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086615:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576072.086617:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576072.086619:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576072.086621:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393257088, offset 0
00000400:00000200:2.0:1788576072.086625:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576072.086629:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576072.086633:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576072.086636:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576072.086638:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576072.086641:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576072.086644:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576072.086647:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331264:331264:256:4294967295] 192.168.202.110@tcp LPNI seq info [331221:331221:8:4294967295]
00000400:00000200:2.0:1788576072.086651:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576072.086657:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576072.086661:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576072.086665:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576072.086669:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576072.086671:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788576072.086675:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576072.086678:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788576072.086683:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576072.086684:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576072.086687:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576072.086690:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576072.086693:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576072.086695:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576072.086697:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576072.086701:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086708:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576072.086711:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576097 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086716:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576072.086718:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576072.086720:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576072.086724:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576072.086726:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576072.086729:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307675687
00000100:00000001:2.0:1788576072.086731:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307676602
00000100:00000001:2.0:1788576072.086733:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307675687
00000100:00000001:2.0:1788576072.086735:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307676602
00000100:00000001:2.0:1788576072.086737:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576072.086739:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576072.086741:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576072.086744:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576072/real 1788576072]  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576072.086752:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576072.086753:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576072.086754:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576072.086756:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576072.086758:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576072.086760:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576072.086763:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576072.086769:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086775:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576072.086788:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.086790:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576072.086792:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576072.086796:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086802:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576072.086805:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086811:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576072.086814:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.086815:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576072.086816:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.086818:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576072.086820:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576072.086822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576072.086826:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576072.086828:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576072.086831:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576072.086838:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576072.086841:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576072.086844:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086850:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576072.086851:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576072.086853:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576072.086855:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576072.086857:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576072.086860:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576072.086863:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576072.086865:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576072.086866:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576072.086868:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576072.086872:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788576072.086873:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576072.086877:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576072.086879:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576072.086883:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576072.086885:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393257216, portal 4
00000100:00000001:2.0:1788576072.086888:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576072.086890:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:2.0:1788576072.086895:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086901:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576072.086904:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576072.086906:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576072.086908:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393257216, offset 0
00000400:00000200:2.0:1788576072.086911:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576072.086915:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576072.086919:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576072.086921:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576072.086924:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576072.086926:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576072.086928:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576072.086931:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331265:331265:256:4294967295] 192.168.202.110@tcp LPNI seq info [331222:331222:8:4294967295]
00000400:00000200:2.0:1788576072.086935:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576072.086941:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576072.086945:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576072.086948:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576072.086951:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576072.086954:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1788576072.086958:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576072.086960:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1788576072.086964:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576072.086965:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576072.086968:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576072.086971:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576072.086974:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576072.086975:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576072.086978:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576072.086981:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576097 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.086994:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576072.086997:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576097 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.087003:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576072.087004:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576072.087007:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576072.087010:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576072.087013:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576072.087015:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307675833
00000100:00000001:2.0:1788576072.087017:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307676602
00000100:00000001:2.0:1788576072.087019:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307675833
00000100:00000001:2.0:1788576072.087021:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307676602
00000100:00000001:2.0:1788576072.087023:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576072.087025:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576072.087027:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576072.087031:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576072/real 1788576072]  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576072.087038:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576072.087040:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576072.087042:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576072.087044:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576072.087045:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576072.087047:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576072.087050:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576072.087059:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.087065:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576072.087066:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.087068:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576072.087069:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576072.087072:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.087079:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576072.087083:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.087089:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576072.087091:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.087093:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576072.087094:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.087096:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576072.087097:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576072.087100:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576072.087102:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576072.087104:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576072.087108:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576072.087115:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576072.087118:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.087120:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576072.087123:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393256960/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.087129:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576072.087132:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576072.087134:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788576072.087137:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576072.087139:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576072.087140:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576072.087142:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576072.087145:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:2.0:1788576072.087149:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576072.087151:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576072.087152:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:2.0:1788576072.087155:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576072.087157:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.087158:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576072.087162:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393257088/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.087168:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576072.087169:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576072.087171:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576072.087173:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576072.087174:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576072.087175:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576072.087177:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576072.087178:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:2.0:1788576072.087180:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576072.087181:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576072.087182:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:2.0:1788576072.087185:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576072.087186:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.087187:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576072.087190:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393257216/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576097 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576072.087196:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576072.087197:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576072.087199:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788576072.087201:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576072.087202:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576072.087203:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576072.087204:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576072.087205:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:2.0:1788576072.087207:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576072.087209:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576072.087210:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:2.0:1788576072.087213:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576072.087214:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.087216:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576072.087223:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576072.087225:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576072.087227:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576072.087228:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576072.469537:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576072.469545:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576072.469551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576072.469552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576072.469556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576072.469557:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576072.469665:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576072.469669:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576072.469672:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576072.597540:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576072.597550:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576072.597552:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576072.597559:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576072.597563:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576072.597567:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576072.597569:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576072.597572:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576072.597575:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576073.045514:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307676602, at 4307676608
00000100:00000040:1.0:1788576073.045514:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307676602, at 4307676608
00000100:00000010:1.0:1788576073.045538:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c660.
00000100:00000010:0.0:1788576073.045544:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cae0.
00010000:00000001:3.0:1788576073.493485:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576073.493491:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576073.493495:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576073.493495:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576073.493497:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576073.493498:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576073.493558:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576073.493561:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576073.493565:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576073.621561:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576073.621568:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576073.621570:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576073.621575:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576073.621579:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576073.621582:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576073.621584:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576073.621595:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576073.621599:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576074.517536:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576074.517542:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576074.517548:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576074.517549:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576074.517552:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576074.517553:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576074.517622:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576074.517626:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576074.517630:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576074.645597:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576074.645608:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576074.645610:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576074.645615:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576074.645619:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576074.645623:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576074.645626:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576074.645628:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576074.645631:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576075.541525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576075.541531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576075.541536:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576075.541538:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576075.541541:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576075.541542:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576075.541668:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576075.541671:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576075.541673:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576075.669592:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576075.669600:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576075.669602:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576075.669607:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576075.669609:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576075.669613:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576075.669616:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576075.669618:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576075.669620:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576076.565529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576076.565538:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576076.565544:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576076.565545:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576076.565549:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576076.565550:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576076.565610:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576076.565615:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576076.565618:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576076.693497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576076.693504:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576076.693506:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576076.693512:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576076.693515:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576076.693519:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576076.693521:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576076.693523:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576076.693526:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576077.205513:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576077.205537:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788576077.205540:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576077.205544:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576077.205548:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576077.205555:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13008
00000400:00000200:0.0:1788576077.205563:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576077.205569:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576077.205573:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576077.205576:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.205580:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576077.205583:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.205585:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576077.205589:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576077.205592:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788576077.205594:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576077.205597:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788576077.205599:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788576077.205602:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788576077.205605:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.205607:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576077.205609:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.205611:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576077.205613:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788576077.205617:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576077.205619:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576077.205622:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576077.205625:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576077.205628:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576077.205630:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576077.205632:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576077.205639:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711880.
00000020:00000040:0.0:1788576077.205642:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788576077.205646:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576077.205647:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576077.205650:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576077.205652:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576077.205656:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576077.205662:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5c00.
00000100:00000001:0.0:1788576077.205669:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576077.205677:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576077.205687:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576077.205691:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576077.205697:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c008ff47] to pc [ptlrpcd_rcv+-1]  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576077.205723:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576077.205726:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576077.205729:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788576077.205730:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576077.205732:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576077.205735:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576077.205737:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13008
00000400:00000200:0.0:1788576077.205741:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576077.205745:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576077.205747:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576077.205751:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.205753:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576077.205755:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576077.205756:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576077.205758:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576077.205761:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1788576077.205762:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1788576077.205763:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788576077.205766:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1788576077.205766:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1788576077.205770:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788576077.205774:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788576077.205776:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788576077.205776:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788576077.205783:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.205786:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576077.205805:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.205810:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788576077.205810:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1788576077.205815:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00000001:2.0:1788576077.205815:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576077.205821:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1788576077.205821:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1788576077.205825:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576077.205825:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576077.205830:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576077.205831:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576077.205837:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1788576077.205837:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393257344:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1788576077.205842:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.205844:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1788576077.205845:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1788576077.205849:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:0.0:1788576077.205850:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1788576077.205853:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1788576077.205854:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576077.205857:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788576077.205858:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576077.205863:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713800.
02000000:00000010:2.0:1788576077.205864:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
00000020:00000040:0.0:1788576077.205867:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788576077.205868:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576077.205874:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576077.205877:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576077.205878:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576077.205883:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1788576077.205884:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
02000000:00000001:0.0:1788576077.205885:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576077.205889:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788576077.205891:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
02000000:00000010:0.0:1788576077.205894:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000200:2.0:1788576077.205898:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393257344, portal 25
00000100:00000001:0.0:1788576077.205899:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.205903:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576077.205906:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523503232 : -130484186048384 : ffff895348711880)
00000100:00100000:0.0:1788576077.205908:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1788576077.205916:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576077.205920:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576077.205924:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.205930:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576077.205933:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee0f5ead] to pc [ptlrpcd_rcv+-1]  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1788576077.205938:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576077.205943:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576077.205947:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393257344, offset 0
00000400:00000200:2.0:1788576077.205957:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788576077.205961:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576077.205964:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788576077.205968:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1788576077.205970:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576077.205973:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576077.205975:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576077.205978:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000400:00000200:2.0:1788576077.205979:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:0.0:1788576077.205983:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13008
00000400:00000200:2.0:1788576077.205987:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576077.205989:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576077.205994:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788576077.205997:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1788576077.205998:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576077.206002:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.206004:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576077.206006:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.206008:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576077.206012:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576077.206014:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000400:00000200:2.0:1788576077.206015:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:0.0:1788576077.206016:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576077.206020:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:2.0:1788576077.206021:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:0.0:1788576077.206023:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788576077.206027:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000200:2.0:1788576077.206029:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331266:331266:256:4294967295] 192.168.202.110@tcp LPNI seq info [331223:331223:8:4294967295]
00000100:00000040:0.0:1788576077.206032:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.206037:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1788576077.206040:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1788576077.206041:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576077.206045:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576077.206047:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788576077.206052:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576077.206058:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576077.206059:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1788576077.206063:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576077.206068:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:2.0:1788576077.206069:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1788576077.206071:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576077.206075:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000010:2.0:1788576077.206076:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000008:00100000:0.0:1788576077.206077:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000800:00000200:2.0:1788576077.206084:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576077.206088:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000008:00000001:0.0:1788576077.206092:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576077.206096:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1788576077.206097:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:0.0:1788576077.206102:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713100.
00000800:00000200:2.1:1788576077.206103:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000040:0.0:1788576077.206106:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788576077.206109:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576077.206110:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576077.206114:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576077.206117:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576077.206120:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1788576077.206121:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000010:0.0:1788576077.206125:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000800:00000010:2.0:1788576077.206125:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000100:00000001:0.0:1788576077.206132:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576077.206134:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:0.0:1788576077.206139:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788576077.206143:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576077.206151:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000100:00000001:0.0:1788576077.206153:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1788576077.206156:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:0.0:1788576077.206157:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.206163:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000040:0.0:1788576077.206165:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002cacf3fd] to pc [ptlrpcd_rcv+-1]  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576077.206171:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576077.206178:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576077.206183:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788576077.206188:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1788576077.206189:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576077.206195:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13018)
00000100:00000040:2.0:1788576077.206195:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576102 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206205:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.206207:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576077.206214:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576077.206218:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576077.206220:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307681721
00000100:00000010:2.0:1788576077.206225:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ce40.
00000100:00000001:2.0:1788576077.206232:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576077.206237:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307681721
00000100:00000010:2.0:1788576077.206240:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c240.
00000100:00000001:2.0:1788576077.206243:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576077.206246:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576077.206248:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.206251:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576077.206255:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576077/real 1788576077]  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576077.206264:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576077.206265:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576077.206267:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576077.206270:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576077.206271:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576077.206273:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576077.206277:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576077.206294:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206300:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576077.206302:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206304:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576077.206307:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576077.206311:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206317:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576077.206321:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206329:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576077.206331:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206333:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576077.206334:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206336:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576077.206338:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576077.206340:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576077.206343:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576077.206345:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576077.206349:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576077.206356:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393257344:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576077.206360:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206362:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576077.206365:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711880 x1875444393257344/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206371:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576077.206373:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576077.206376:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576077.206378:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576077.206380:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576077.206382:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576077.206384:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576077.206386:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5c00.
02000000:00000001:2.0:1788576077.206390:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576077.206416:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576077.206418:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711880.
00000100:00000001:2.0:1788576077.206421:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576077.206422:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206424:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206431:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576077.206432:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.206434:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576077.206435:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576077.206437:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576077.206440:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206447:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576077.206448:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576077.206450:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576077.206452:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576077.206454:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576077.206456:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257472:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576077.206459:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576077.206461:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576077.206463:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576077.206465:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576077.206467:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576077.206469:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576077.206473:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576077.206475:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576077.206479:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576077.206481:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393257472, portal 10
00000100:00000001:2.0:1788576077.206484:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576077.206485:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523511296 : -130484186040320 : ffff895348713800)
00000100:00000040:2.0:1788576077.206491:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206497:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576077.206499:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576077.206501:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576077.206503:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393257472, offset 0
00000400:00000200:2.0:1788576077.206507:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576077.206511:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576077.206515:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576077.206517:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576077.206520:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576077.206522:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576077.206524:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576077.206527:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331267:331267:256:4294967295] 192.168.202.110@tcp LPNI seq info [331224:331224:8:4294967295]
00000400:00000200:2.0:1788576077.206530:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576077.206536:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576077.206551:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576077.206554:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576077.206558:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576077.206560:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788576077.206565:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576077.206567:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788576077.206571:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576077.206573:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576077.206575:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576077.206579:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576077.206582:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576077.206584:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576077.206586:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576077.206589:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206597:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576077.206600:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576102 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206606:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.206607:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576077.206610:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576077.206614:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576077.206617:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576077.206619:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307680807
00000100:00000001:2.0:1788576077.206622:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307681721
00000100:00000001:2.0:1788576077.206624:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307680807
00000100:00000001:2.0:1788576077.206626:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307681721
00000100:00000001:2.0:1788576077.206628:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576077.206630:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.206631:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576077.206635:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576077/real 1788576077]  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576077.206641:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576077.206643:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576077.206644:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576077.206647:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576077.206648:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576077.206650:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576077.206653:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576077.206661:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206666:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576077.206668:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206669:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576077.206671:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576077.206674:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206680:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576077.206684:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206690:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576077.206692:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206694:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576077.206695:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206696:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576077.206698:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576077.206700:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576077.206703:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576077.206704:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576077.206707:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576077.206714:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257472:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576077.206717:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576077.206720:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206725:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576077.206726:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576077.206728:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576077.206730:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576077.206732:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576077.206734:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257600:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576077.206737:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576077.206739:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576077.206741:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576077.206743:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576077.206746:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788576077.206748:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576077.206751:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576077.206754:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576077.206757:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576077.206759:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393257600, portal 4
00000100:00000001:2.0:1788576077.206761:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576077.206763:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523509504 : -130484186042112 : ffff895348713100)
00000100:00000040:2.0:1788576077.206768:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206774:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576077.206776:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576077.206778:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576077.206780:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393257600, offset 0
00000400:00000200:2.0:1788576077.206783:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576077.206787:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576077.206803:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576077.206805:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576077.206808:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576077.206810:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576077.206812:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576077.206815:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331268:331268:256:4294967295] 192.168.202.110@tcp LPNI seq info [331225:331225:8:4294967295]
00000400:00000200:2.0:1788576077.206819:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576077.206824:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576077.206828:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576077.206831:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576077.206834:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576077.206837:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788576077.206841:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576077.206843:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788576077.206847:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576077.206849:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576077.206851:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576077.206854:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576077.206856:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576077.206858:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576077.206860:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576077.206863:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576102 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206869:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576077.206873:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576102 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206878:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.206880:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576077.206882:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576077.206884:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576077.206887:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576077.206889:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307680953
00000100:00000001:2.0:1788576077.206891:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307681721
00000100:00000001:2.0:1788576077.206893:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307680953
00000100:00000001:2.0:1788576077.206895:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307681721
00000100:00000001:2.0:1788576077.206897:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576077.206898:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.206900:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576077.206903:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576077/real 1788576077]  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576077.206909:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576077.206910:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576077.206911:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576077.206913:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576077.206915:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576077.206917:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576077.206919:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576077.206927:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206933:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576077.206934:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206936:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576077.206937:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576077.206940:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206947:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576077.206950:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206956:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576077.206959:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206960:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576077.206961:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206963:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576077.206965:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576077.206968:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576077.206970:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576077.206973:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576077.206976:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576077.206984:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257600:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576077.206988:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.206989:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576077.206992:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713800 x1875444393257472/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.206997:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576077.206999:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576077.207000:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576077.207003:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576077.207005:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576077.207006:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576077.207008:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576077.207010:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:2.0:1788576077.207012:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576077.207014:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576077.207015:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713800.
00000100:00000001:2.0:1788576077.207018:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576077.207019:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.207020:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576077.207023:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713100 x1875444393257600/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576102 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576077.207028:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576077.207030:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576077.207031:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788576077.207033:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576077.207034:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576077.207035:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576077.207037:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576077.207038:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788576077.207040:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576077.207041:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576077.207042:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713100.
00000100:00000001:2.0:1788576077.207044:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576077.207045:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.207046:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576077.207049:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576077.207050:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576077.207052:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576077.207053:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576077.589510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576077.589516:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576077.589523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576077.589526:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576077.589528:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576077.589532:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576077.589535:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576077.589537:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576077.589539:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576077.589541:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576077.589544:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576077.589546:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576077.589547:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576077.589549:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576077.589550:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576077.589552:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576077.589553:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576077.589555:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576077.589611:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576077.589615:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576077.589619:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000008:00000020:2.0:1788576077.717493:0:216532:0:(osc_request.c:745:osc_update_next_shrink()) next time 14213 to shrink grant
00000100:00000001:0.0:1788576077.717497:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000008:00000020:2.0:1788576077.717502:0:216532:0:(osc_request.c:745:osc_update_next_shrink()) next time 14213 to shrink grant
00000100:00000001:0.0:1788576077.717503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576077.717505:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576077.717509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576077.717512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576077.717515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576077.717516:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576077.717518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576077.717520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576078.165504:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307681721, at 4307681728
00000100:00000010:0.0:1788576078.165521:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ce40.
00000100:00000040:1.0:1788576078.165541:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307681721, at 4307681728
00000100:00000010:1.0:1788576078.165550:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c240.
00010000:00000001:3.0:1788576078.614464:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576078.614471:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576078.614476:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576078.614477:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576078.614480:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576078.614481:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576078.615191:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576078.615195:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576078.615199:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576078.741499:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576078.741506:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576078.741509:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576078.741514:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576078.741518:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576078.741522:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576078.741524:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576078.741527:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576078.741530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576079.638700:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576079.638708:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576079.638711:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576079.638714:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576079.638728:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576079.638731:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576079.638734:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576079.638736:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576079.638740:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576079.638741:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576079.638745:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576079.638746:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576079.638844:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576079.638848:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576079.638851:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576079.765956:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576079.765965:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576079.765967:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576079.765973:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576079.765976:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576079.765980:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576079.765983:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576079.765985:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576079.765988:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576080.661524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576080.661539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576080.661544:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576080.661545:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576080.661548:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576080.661550:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576080.661615:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576080.661620:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576080.661624:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576080.789513:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576080.789520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576080.789522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576080.789527:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576080.789531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576080.789535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576080.789537:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576080.789540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576080.789542:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576081.687740:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576081.687748:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576081.687754:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576081.687755:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576081.687759:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576081.687760:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576081.687882:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576081.687885:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576081.687888:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576081.813517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576081.813524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576081.813526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576081.813531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576081.813535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576081.813539:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576081.813541:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576081.813543:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576081.813546:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576082.326958:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576082.326971:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576082.326973:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576082.326978:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576082.326985:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576082.326992:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13013
00000400:00000200:3.0:1788576082.327001:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576082.327006:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576082.327009:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576082.327011:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327014:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576082.327015:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327017:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576082.327020:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576082.327021:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576082.327024:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576082.327027:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576082.327028:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576082.327030:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576082.327032:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327033:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576082.327034:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327035:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576082.327037:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576082.327039:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327041:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576082.327043:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576082.327045:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576082.327047:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576082.327049:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576082.327051:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576082.327059:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2a00.
00000020:00000040:3.0:1788576082.327061:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576082.327064:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576082.327065:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576082.327068:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576082.327070:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576082.327073:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576082.327079:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788576082.327084:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576082.327090:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576082.327098:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576082.327101:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576082.327104:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f91f0d0] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576082.327122:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576082.327125:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576082.327128:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576082.327129:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576082.327130:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576082.327132:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576082.327133:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13013
00000400:00000200:3.0:1788576082.327136:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576082.327138:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576082.327139:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576082.327141:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327142:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576082.327144:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327145:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576082.327146:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576082.327147:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576082.327148:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576082.327150:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788576082.327151:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576082.327152:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788576082.327153:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327154:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576082.327155:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327156:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576082.327158:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788576082.327159:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327160:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576082.327161:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576082.327165:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576082.327166:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576082.327167:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576082.327171:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788576082.327173:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576082.327174:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576082.327176:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2680.
00000020:00000040:3.0:1788576082.327177:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788576082.327179:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576082.327180:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576082.327182:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576082.327183:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576082.327185:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576082.327188:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788576082.327191:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576082.327196:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576082.327202:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576082.327204:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576082.327207:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026a391d1] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576082.327214:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576082.327216:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576082.327219:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788576082.327221:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576082.327222:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576082.327225:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576082.327227:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13013
00000400:00000200:3.0:1788576082.327232:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576082.327234:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576082.327237:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576082.327239:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327241:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576082.327243:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327244:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576082.327246:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576082.327246:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576082.327247:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576082.327249:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788576082.327250:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788576082.327251:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788576082.327253:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327254:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576082.327255:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327256:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576082.327257:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788576082.327258:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576082.327260:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576082.327261:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576082.327263:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576082.327264:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576082.327265:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576082.327266:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788576082.327268:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576082.327269:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576082.327271:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3b80.
00000020:00000040:3.0:1788576082.327273:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788576082.327274:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576082.327275:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576082.327276:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576082.327277:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576082.327278:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576082.327280:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788576082.327282:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576082.327285:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576082.327294:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576082.327295:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576082.327297:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061a6ac79] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576082.327301:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576082.327303:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788576082.327305:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788576082.327308:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13023)
00000100:00000001:2.0:1788576082.327359:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576082.327362:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576082.327364:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576082.327369:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327374:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576082.327376:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576082.327379:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576082.327380:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576082.327382:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576082.327384:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393257728:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576082.327388:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576082.327412:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576082.327415:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576082.327418:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576082.327423:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576082.327425:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576082.327431:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576082.327434:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576082.327441:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576082.327447:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393257728, portal 25
00000100:00000001:2.0:1788576082.327450:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576082.327451:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138825216 : -130483570726400 : ffff89536d1e2a00)
00000100:00000040:2.0:1788576082.327456:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327463:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576082.327466:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576082.327468:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576082.327472:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393257728, offset 0
00000400:00000200:2.0:1788576082.327478:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576082.327485:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576082.327492:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576082.327497:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576082.327500:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576082.327504:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576082.327507:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576082.327510:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331269:331269:256:4294967295] 192.168.202.110@tcp LPNI seq info [331226:331226:8:4294967295]
00000400:00000200:2.0:1788576082.327515:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576082.327524:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576082.327531:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576082.327536:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576082.327543:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576082.327547:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788576082.327552:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576082.327555:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788576082.327560:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576082.327562:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576082.327568:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576082.327572:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576082.327578:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576082.327581:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576082.327585:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576082.327589:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327597:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576082.327600:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576107 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327606:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576082.327608:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576082.327614:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576082.327618:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576082.327621:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307686843
00000100:00000010:2.0:1788576082.327625:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd20.
00000100:00000001:2.0:1788576082.327631:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576082.327636:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307686843
00000100:00000010:2.0:1788576082.327639:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c360.
00000100:00000001:2.0:1788576082.327642:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576082.327645:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576082.327647:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576082.327650:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576082.327654:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576082/real 1788576082]  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576082.327668:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576082.327670:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576082.327672:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576082.327674:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576082.327676:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576082.327678:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576082.327681:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576082.327690:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327696:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576082.327697:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.327700:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576082.327701:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576082.327705:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327711:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576082.327715:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327721:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576082.327724:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.327725:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576082.327726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.327728:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576082.327730:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576082.327734:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576082.327737:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576082.327740:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576082.327744:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576082.327752:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393257728:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576082.327755:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576082.327757:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327761:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576082.327762:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576082.327764:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576082.327765:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576082.327766:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576082.327768:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257856:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576082.327770:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576082.327771:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576082.327772:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576082.327773:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576082.327776:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788576082.327777:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576082.327780:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576082.327781:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576082.327784:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576082.327786:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393257856, portal 10
00000100:00000001:2.0:1788576082.327787:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576082.327788:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138824320 : -130483570727296 : ffff89536d1e2680)
00000100:00000040:2.0:1788576082.327792:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327796:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576082.327797:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576082.327799:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576082.327800:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393257856, offset 0
00000400:00000200:2.0:1788576082.327833:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576082.327837:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576082.327839:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576082.327841:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576082.327843:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576082.327844:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576082.327846:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576082.327848:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331270:331270:256:4294967295] 192.168.202.110@tcp LPNI seq info [331227:331227:8:4294967295]
00000400:00000200:2.0:1788576082.327850:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576082.327854:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576082.327857:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576082.327860:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576082.327862:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576082.327864:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788576082.327867:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576082.327868:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788576082.327871:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576082.327872:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576082.327874:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576082.327876:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576082.327878:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576082.327879:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576082.327881:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576082.327883:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327889:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576082.327890:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576107 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327894:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576082.327895:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576082.327897:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576082.327899:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576082.327901:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576082.327903:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307685928
00000100:00000001:2.0:1788576082.327905:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307686843
00000100:00000001:2.0:1788576082.327906:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307685928
00000100:00000001:2.0:1788576082.327908:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307686843
00000100:00000001:2.0:1788576082.327909:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576082.327910:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576082.327911:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576082.327913:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576082/real 1788576082]  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576082.327917:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576082.327918:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576082.327919:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576082.327921:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576082.327922:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576082.327923:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576082.327925:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576082.327930:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327933:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576082.327934:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.327935:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576082.327936:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576082.327938:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327941:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576082.327943:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327947:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576082.327948:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.327949:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576082.327950:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.327951:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576082.327953:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576082.327954:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576082.327956:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576082.327957:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576082.327959:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576082.327963:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257856:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576082.327965:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576082.327967:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.327971:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576082.327972:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576082.327973:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576082.327974:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576082.327975:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576082.327977:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576082.327978:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576082.327980:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576082.327981:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576082.327982:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576082.327984:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576082.327985:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576082.327987:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576082.327988:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576082.327990:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576082.327992:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393257984, portal 4
00000100:00000001:2.0:1788576082.327994:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576082.327995:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138829696 : -130483570721920 : ffff89536d1e3b80)
00000100:00000040:2.0:1788576082.327998:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.328002:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576082.328003:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576082.328004:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576082.328005:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393257984, offset 0
00000400:00000200:2.0:1788576082.328008:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576082.328011:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576082.328014:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576082.328017:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576082.328019:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576082.328021:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576082.328023:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576082.328026:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331271:331271:256:4294967295] 192.168.202.110@tcp LPNI seq info [331228:331228:8:4294967295]
00000400:00000200:2.0:1788576082.328030:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576082.328035:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576082.328039:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576082.328042:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576082.328045:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576082.328048:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788576082.328051:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576082.328053:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788576082.328056:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576082.328058:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576082.328060:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576082.328062:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576082.328065:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576082.328066:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576082.328068:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576082.328071:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576107 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.328083:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576082.328085:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576107 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.328091:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576082.328092:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576082.328094:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576082.328097:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576082.328100:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576082.328101:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307686074
00000100:00000001:2.0:1788576082.328103:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307686843
00000100:00000001:2.0:1788576082.328105:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307686074
00000100:00000001:2.0:1788576082.328108:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307686843
00000100:00000001:2.0:1788576082.328109:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576082.328111:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576082.328113:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576082.328117:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576082/real 1788576082]  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576082.328124:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576082.328125:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576082.328126:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576082.328129:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576082.328130:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576082.328132:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576082.328135:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576082.328142:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.328148:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576082.328149:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.328151:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576082.328152:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576082.328155:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.328161:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576082.328165:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.328171:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576082.328174:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.328175:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576082.328176:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.328178:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576082.328180:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576082.328183:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576082.328186:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576082.328188:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576082.328192:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576082.328198:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393257984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576082.328202:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.328204:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576082.328207:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2a00 x1875444393257728/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.328213:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576082.328215:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576082.328217:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576082.328220:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576082.328221:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576082.328223:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576082.328225:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576082.328228:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788576082.328231:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576082.328232:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576082.328234:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2a00.
00000100:00000001:2.0:1788576082.328237:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576082.328238:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.328239:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576082.328242:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2680 x1875444393257856/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.328247:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576082.328248:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576082.328250:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788576082.328251:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576082.328252:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576082.328253:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576082.328255:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576082.328256:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788576082.328258:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576082.328259:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576082.328260:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2680.
00000100:00000001:2.0:1788576082.328261:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576082.328262:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.328264:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576082.328267:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3b80 x1875444393257984/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576107 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576082.328273:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576082.328274:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576082.328275:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576082.328277:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576082.328278:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576082.328279:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576082.328281:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576082.328282:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788576082.328285:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576082.328286:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576082.328287:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3b80.
00000100:00000001:2.0:1788576082.328290:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576082.328291:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.328293:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576082.328301:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576082.328303:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576082.328305:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576082.328307:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576082.709512:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576082.709519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576082.709525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576082.709526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576082.709530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576082.709531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576082.710086:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576082.710091:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576082.710094:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576082.837517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576082.837524:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576082.837527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576082.837532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576082.837536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576082.837540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576082.837543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576082.837546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576082.837548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576083.285506:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307686843, at 4307686848
00000100:00000010:1.0:1788576083.285522:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c360.
00000100:00000040:0.0:1788576083.286881:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307686843, at 4307686849
00000100:00000010:0.0:1788576083.286889:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd20.
00010000:00000001:3.0:1788576083.733587:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576083.733594:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576083.733600:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576083.733601:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576083.733604:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576083.733606:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576083.733652:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576083.733656:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576083.733658:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576083.861512:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576083.861519:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576083.861532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576083.861538:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576083.861541:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576083.861545:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576083.861548:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576083.861551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576083.861554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576084.757503:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576084.757510:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576084.757516:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576084.757517:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576084.757520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576084.757522:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576084.757585:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576084.757589:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576084.757592:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576084.887676:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576084.887683:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576084.887685:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576084.887690:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576084.887693:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576084.887696:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576084.887699:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576084.887701:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576084.887704:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576085.781489:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576085.781496:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576085.781502:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576085.781503:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576085.781507:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576085.781508:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576085.781557:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576085.781561:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576085.781564:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576085.909495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576085.909502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576085.909504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576085.909509:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576085.909512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576085.909516:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576085.909519:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576085.909522:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576085.909525:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576086.805602:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576086.805609:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576086.805614:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576086.805617:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576086.805619:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576086.805623:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576086.805626:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576086.805628:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576086.805630:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576086.805632:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576086.805636:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576086.805637:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576086.805638:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576086.805639:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576086.805641:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576086.805642:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576086.805644:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576086.805646:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576086.805738:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576086.805742:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576086.805745:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576086.933530:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576086.933537:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576086.933539:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576086.933543:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576086.933546:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576086.933549:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576086.933551:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576086.933553:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576086.933555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576087.445522:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576087.445532:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576087.445534:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576087.445538:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576087.445542:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576087.445550:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13018
00000400:00000200:3.0:1788576087.445560:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576087.445566:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576087.445569:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576087.445573:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445575:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576087.445578:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445581:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576087.445585:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576087.445587:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576087.445590:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576087.445593:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576087.445596:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576087.445598:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576087.445601:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445603:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576087.445605:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445607:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576087.445609:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576087.445613:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445615:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576087.445618:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576087.445621:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576087.445624:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576087.445626:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576087.445628:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576087.445638:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1500.
00000020:00000040:3.0:1788576087.445642:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576087.445646:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576087.445647:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576087.445650:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576087.445653:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576087.445658:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576087.445665:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788576087.445672:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576087.445681:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576087.445692:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576087.445696:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576087.445700:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035452eb3] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576087.445726:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576087.445729:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576087.445733:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576087.445734:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576087.445735:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576087.445738:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576087.445740:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13018
00000400:00000200:3.0:1788576087.445744:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576087.445747:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576087.445749:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576087.445751:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445754:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576087.445755:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445758:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576087.445760:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576087.445761:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576087.445763:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576087.445765:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788576087.445767:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576087.445768:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788576087.445771:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445773:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576087.445774:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445776:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576087.445777:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788576087.445779:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445780:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576087.445781:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576087.445785:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576087.445786:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576087.445787:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576087.445790:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788576087.445793:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576087.445794:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576087.445796:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1f80.
00000020:00000040:3.0:1788576087.445798:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788576087.445800:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576087.445801:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576087.445802:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576087.445803:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576087.445804:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576087.445806:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788576087.445809:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576087.445812:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576087.445828:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576087.445830:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576087.445834:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ff7e543] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576087.445842:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576087.445845:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576087.445851:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788576087.445853:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576087.445854:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576087.445857:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576087.445859:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13018
00000400:00000200:3.0:1788576087.445863:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576087.445865:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576087.445867:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576087.445870:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445871:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576087.445873:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445876:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576087.445878:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576087.445879:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576087.445881:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576087.445883:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788576087.445885:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788576087.445886:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788576087.445889:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445890:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576087.445893:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445895:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576087.445896:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788576087.445899:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576087.445901:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576087.445903:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576087.445905:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576087.445907:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576087.445909:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576087.445910:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788576087.445913:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576087.445914:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576087.445917:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3800.
00000020:00000040:3.0:1788576087.445919:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788576087.445922:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576087.445923:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576087.445924:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576087.445926:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576087.445927:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576087.445931:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788576087.445934:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576087.445938:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576087.445951:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576087.445953:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576087.445957:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df97a4b] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576087.445963:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576087.445965:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788576087.445969:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788576087.445973:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13028)
00000100:00000001:2.0:1788576087.446065:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576087.446070:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576087.446072:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576087.446078:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446086:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576087.446088:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576087.446090:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576087.446092:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576087.446095:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576087.446098:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393258112:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576087.446101:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576087.446105:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576087.446107:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576087.446110:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576087.446114:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576087.446116:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576087.446121:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576087.446125:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576087.446130:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576087.446137:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393258112, portal 25
00000100:00000001:2.0:1788576087.446140:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576087.446141:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138819840 : -130483570731776 : ffff89536d1e1500)
00000100:00000040:2.0:1788576087.446146:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446153:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576087.446156:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576087.446159:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576087.446163:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393258112, offset 0
00000400:00000200:2.0:1788576087.446169:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576087.446176:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576087.446182:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576087.446187:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576087.446191:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576087.446195:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576087.446197:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576087.446201:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331272:331272:256:4294967295] 192.168.202.110@tcp LPNI seq info [331229:331229:8:4294967295]
00000400:00000200:2.0:1788576087.446205:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576087.446214:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576087.446219:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576087.446223:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576087.446229:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576087.446233:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788576087.446238:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576087.446241:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788576087.446245:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576087.446248:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576087.446255:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576087.446259:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576087.446264:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576087.446267:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576087.446271:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576087.446274:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446283:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576087.446286:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446292:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576087.446294:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576087.446300:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576087.446304:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576087.446306:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307691961
00000100:00000010:2.0:1788576087.446311:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c960.
00000100:00000001:2.0:1788576087.446317:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576087.446323:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307691961
00000100:00000010:2.0:1788576087.446326:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c8a0.
00000100:00000001:2.0:1788576087.446329:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576087.446332:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576087.446335:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576087.446337:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576087.446341:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576087/real 1788576087]  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576087.446350:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576087.446352:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576087.446354:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576087.446357:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576087.446358:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576087.446361:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576087.446364:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576087.446372:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446378:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576087.446379:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.446381:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576087.446383:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576087.446386:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446414:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576087.446418:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446426:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576087.446429:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.446431:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576087.446432:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.446434:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576087.446436:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576087.446439:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576087.446442:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576087.446444:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576087.446448:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576087.446455:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393258112:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576087.446459:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576087.446462:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446468:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576087.446469:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576087.446471:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576087.446472:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576087.446474:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576087.446477:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393258240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576087.446480:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576087.446482:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576087.446483:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576087.446484:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576087.446487:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788576087.446489:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576087.446492:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576087.446495:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576087.446498:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576087.446502:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393258240, portal 10
00000100:00000001:2.0:1788576087.446504:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576087.446505:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138822528 : -130483570729088 : ffff89536d1e1f80)
00000100:00000040:2.0:1788576087.446510:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446517:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576087.446520:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576087.446522:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576087.446525:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393258240, offset 0
00000400:00000200:2.0:1788576087.446528:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576087.446531:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576087.446535:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576087.446538:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576087.446540:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576087.446542:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576087.446544:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576087.446546:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331273:331273:256:4294967295] 192.168.202.110@tcp LPNI seq info [331230:331230:8:4294967295]
00000400:00000200:2.0:1788576087.446550:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576087.446556:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576087.446560:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576087.446563:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576087.446567:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576087.446569:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788576087.446573:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576087.446575:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788576087.446579:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576087.446580:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576087.446583:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576087.446586:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576087.446588:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576087.446590:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576087.446593:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576087.446596:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446603:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576087.446605:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446611:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576087.446613:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576087.446616:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576087.446619:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576087.446622:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576087.446624:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307691047
00000100:00000001:2.0:1788576087.446626:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307691961
00000100:00000001:2.0:1788576087.446628:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307691047
00000100:00000001:2.0:1788576087.446630:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307691961
00000100:00000001:2.0:1788576087.446632:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576087.446634:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576087.446636:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576087.446639:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576087/real 1788576087]  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576087.446646:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576087.446647:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576087.446649:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576087.446651:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576087.446653:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576087.446655:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576087.446658:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576087.446665:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446676:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576087.446677:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.446678:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576087.446679:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576087.446682:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446687:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576087.446690:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446695:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576087.446697:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.446698:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576087.446699:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.446701:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576087.446703:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576087.446705:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576087.446707:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576087.446709:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576087.446712:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576087.446718:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393258240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576087.446721:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576087.446723:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446729:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576087.446730:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576087.446732:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576087.446733:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576087.446735:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576087.446737:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393258368:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576087.446741:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576087.446742:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576087.446744:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576087.446745:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576087.446748:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576087.446749:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576087.446752:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576087.446754:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576087.446757:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576087.446758:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393258368, portal 4
00000100:00000001:2.0:1788576087.446760:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576087.446761:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138828800 : -130483570722816 : ffff89536d1e3800)
00000100:00000040:2.0:1788576087.446766:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446770:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576087.446772:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576087.446774:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576087.446775:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393258368, offset 0
00000400:00000200:2.0:1788576087.446779:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576087.446781:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576087.446784:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576087.446787:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576087.446789:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576087.446790:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576087.446792:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576087.446795:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331274:331274:256:4294967295] 192.168.202.110@tcp LPNI seq info [331231:331231:8:4294967295]
00000400:00000200:2.0:1788576087.446798:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576087.446803:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576087.446808:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576087.446810:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576087.446813:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576087.446939:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788576087.446944:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576087.446947:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788576087.446950:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576087.446951:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576087.446954:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576087.446957:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576087.446959:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576087.446960:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576087.446962:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576087.446965:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446971:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576087.446974:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.446978:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576087.446980:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576087.446982:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576087.446984:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576087.446987:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576087.446989:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307691193
00000100:00000001:2.0:1788576087.446991:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307691961
00000100:00000001:2.0:1788576087.446993:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307691193
00000100:00000001:2.0:1788576087.446995:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307691961
00000100:00000001:2.0:1788576087.446996:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576087.446998:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576087.446999:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576087.447002:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576087/real 1788576087]  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576087.447009:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576087.447010:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576087.447012:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576087.447013:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576087.447015:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576087.447016:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576087.447019:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576087.447026:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.447031:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576087.447033:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.447034:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576087.447035:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576087.447038:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.447043:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576087.447046:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.447051:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576087.447053:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.447054:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576087.447055:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.447057:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576087.447058:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576087.447060:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576087.447062:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576087.447064:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576087.447068:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576087.447073:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393258368:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576087.447077:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.447078:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576087.447081:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1500 x1875444393258112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.447086:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576087.447088:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576087.447090:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576087.447092:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576087.447094:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576087.447095:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576087.447097:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576087.447099:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788576087.447102:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576087.447104:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576087.447105:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1500.
00000100:00000001:2.0:1788576087.447108:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576087.447109:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.447111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576087.447114:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1f80 x1875444393258240/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.447121:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576087.447122:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576087.447124:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11000.
02000000:00000001:2.0:1788576087.447126:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576087.447127:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576087.447128:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576087.447130:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576087.447132:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788576087.447134:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576087.447135:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576087.447136:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1f80.
00000100:00000001:2.0:1788576087.447139:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576087.447140:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.447141:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576087.447144:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3800 x1875444393258368/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576087.447150:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576087.447152:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576087.447153:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576087.447155:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576087.447156:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576087.447157:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576087.447159:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576087.447160:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788576087.447162:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576087.447163:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576087.447165:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3800.
00000100:00000001:2.0:1788576087.447167:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576087.447169:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.447171:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576087.447177:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576087.447179:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576087.447182:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576087.447183:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576087.831040:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576087.831047:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576087.831053:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576087.831054:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576087.831058:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576087.831059:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576087.831129:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576087.831133:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576087.831137:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576087.957457:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576087.957464:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576087.957466:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576087.957471:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576087.957474:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576087.957478:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576087.957481:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576087.957484:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576087.957487:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576088.405501:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307691961, at 4307691968
00000100:00000010:1.0:1788576088.405516:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c8a0.
00000100:00000040:0.0:1788576088.405520:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307691961, at 4307691968
00000100:00000010:0.0:1788576088.405530:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c960.
00010000:00000001:3.0:1788576088.854002:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576088.854011:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576088.854013:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576088.854016:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576088.854020:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576088.854022:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576088.854024:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576088.854026:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576088.854030:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576088.854031:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576088.854034:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576088.854036:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576088.854246:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576088.854250:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576088.854253:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576088.981583:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576088.981597:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576088.981599:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576088.981604:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576088.981608:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576088.981612:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576088.981614:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576088.981616:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576088.981619:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576089.877531:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576089.877539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576089.877545:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576089.877546:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576089.877550:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576089.877551:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576089.877703:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576089.877708:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576089.877711:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576090.005549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576090.005558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576090.005560:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576090.005566:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576090.005569:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576090.005573:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576090.005576:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576090.005579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576090.005582:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576090.901659:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576090.901669:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576090.901677:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576090.901678:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576090.901683:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576090.901684:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576090.902785:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576090.902791:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576090.902795:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576091.029536:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576091.029543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576091.029545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576091.029557:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576091.029561:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576091.029565:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576091.029568:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576091.029570:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576091.029573:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576091.925569:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576091.925579:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576091.925586:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576091.925588:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576091.925592:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576091.925593:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576091.925715:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576091.925720:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576091.925724:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576092.053646:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576092.053658:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576092.053661:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576092.053668:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576092.053672:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576092.053678:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.053682:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576092.053684:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.053687:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576092.565599:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576092.565607:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788576092.565615:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576092.565618:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576092.565622:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576092.565627:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13023
00000400:00000200:2.0:1788576092.565634:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576092.565639:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576092.565642:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576092.565644:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565647:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576092.565649:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565650:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576092.565654:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576092.565655:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788576092.565658:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576092.565660:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788576092.565662:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788576092.565663:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788576092.565665:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565666:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576092.565667:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565668:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576092.565670:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788576092.565672:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565673:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576092.565675:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576092.565678:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576092.565680:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576092.565681:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788576092.565683:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576092.565691:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02680.
00000020:00000040:2.0:1788576092.565694:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788576092.565697:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576092.565698:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576092.565700:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576092.565702:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576092.565706:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576092.565711:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788576092.565716:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576092.565724:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576092.565731:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576092.565735:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576092.565738:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000450d1599] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576092.565757:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576092.565759:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576092.565762:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788576092.565763:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576092.565764:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576092.565765:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576092.565767:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13023
00000400:00000200:2.0:1788576092.565771:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576092.565774:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576092.565777:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576092.565780:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565784:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576092.565787:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565789:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576092.565791:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576092.565792:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788576092.565794:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788576092.565798:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1788576092.565799:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788576092.565803:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788576092.565809:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788576092.565810:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576092.565816:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576092.565818:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565823:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576092.565825:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565860:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576092.565863:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788576092.565876:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576092.565880:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576092.565881:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576092.565887:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576092.565888:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576092.565997:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576092.566001:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788576092.566003:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576092.566004:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576092.566007:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e00700.
00000020:00000040:2.0:1788576092.566009:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788576092.566010:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576092.566011:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576092.566013:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576092.566014:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576092.566015:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576092.566018:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788576092.566021:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576092.566026:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576092.566030:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576092.566032:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576092.566034:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e69941e] to pc [ptlrpcd_rcv+-1]  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576092.566048:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576092.566050:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576092.566053:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788576092.566054:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576092.566055:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576092.566057:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576092.566058:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13023
00000400:00000200:2.0:1788576092.566061:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576092.566063:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576092.566064:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576092.566066:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.566068:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576092.566069:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.566070:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576092.566071:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576092.566072:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788576092.566073:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576092.566074:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788576092.566075:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788576092.566076:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788576092.566078:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.566079:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576092.566080:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576092.566081:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576092.566083:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788576092.566084:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576092.566086:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576092.566087:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576092.566088:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576092.566089:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576092.566091:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576092.566092:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788576092.566093:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576092.566094:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576092.566096:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000020:00000040:2.0:1788576092.566097:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788576092.566099:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576092.566099:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576092.566100:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576092.566102:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576092.566103:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576092.566105:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11800.
00000100:00000001:2.0:1788576092.566107:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576092.566109:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576092.566114:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576092.566115:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576092.566117:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576092.566122:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576092.566123:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788576092.566126:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788576092.566129:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13033)
00000100:00000040:0.0:1788576092.566189:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566204:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576092.566208:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576092.566214:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576092.566218:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576092.566220:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576092.566225:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393258496:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576092.566230:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576092.566237:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576092.566241:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576092.566244:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576092.566254:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576092.566257:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576092.566265:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576092.566271:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576092.566280:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576092.566288:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393258496, portal 25
00000100:00000001:0.0:1788576092.566291:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576092.566293:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530781312 : -130484178770304 : ffff895348e02680)
00000100:00000040:0.0:1788576092.566300:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576117 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566309:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576092.566313:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576092.566316:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576092.566319:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393258496, offset 0
00000400:00000200:0.0:1788576092.566327:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576092.566338:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576092.566346:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576092.566353:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576092.566359:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576092.566363:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576092.566366:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576092.566371:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331275:331275:256:4294967295] 192.168.202.110@tcp LPNI seq info [331232:331232:8:4294967295]
00000400:00000200:0.0:1788576092.566376:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576092.566387:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576092.566423:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576092.566430:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576092.566438:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576092.566442:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788576092.566449:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576092.566453:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788576092.566458:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576092.566463:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576092.566472:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576092.566477:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576092.566485:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576092.566488:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576092.566493:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576092.566497:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576117 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566507:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576092.566510:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576117 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566519:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576092.566521:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576092.566528:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576092.566533:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576092.566536:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307697082
00000100:00000010:0.0:1788576092.566542:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576092.566551:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576092.566558:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307697082
00000100:00000010:0.0:1788576092.566562:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7180.
00000100:00000001:0.0:1788576092.566565:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576092.566569:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576092.566572:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576092.566575:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576092.566580:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576092/real 1788576092]  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576092.566592:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576092.566594:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576092.566596:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576092.566599:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576092.566601:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576092.566604:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576092.566608:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576092.566617:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566623:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576092.566625:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.566628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576092.566630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576092.566658:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566666:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576092.566673:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566681:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576092.566685:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.566686:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576092.566688:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.566691:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576092.566693:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576092.566697:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576092.566701:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576092.566704:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576092.566708:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576092.566716:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393258496:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576092.566722:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.566724:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576092.566727:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02680 x1875444393258496/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566741:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576092.566744:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576092.566747:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576092.566752:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576092.566756:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576092.566758:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576092.566760:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576092.566764:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:0.0:1788576092.566768:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576092.566770:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576092.566772:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02680.
00000100:00000001:0.0:1788576092.566776:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576092.566777:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.566780:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.566788:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576092.566790:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576092.566792:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576092.566794:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576092.566795:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576092.566799:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566805:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576092.566807:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576092.566809:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576092.566811:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576092.566813:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576092.566816:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393258624:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576092.566819:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576092.566821:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576092.566823:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576092.566825:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576092.566839:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576092.566842:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576092.566845:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576092.566849:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576092.566853:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576092.566856:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393258624, portal 10
00000100:00000001:0.0:1788576092.566858:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576092.566859:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530773248 : -130484178778368 : ffff895348e00700)
00000100:00000040:0.0:1788576092.566866:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576117 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566879:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576092.566881:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576092.566884:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576092.566886:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393258624, offset 0
00000400:00000200:0.0:1788576092.566891:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576092.566895:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576092.566898:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576092.566901:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576092.566904:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576092.566906:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576092.566909:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576092.566912:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331276:331276:256:4294967295] 192.168.202.110@tcp LPNI seq info [331233:331233:8:4294967295]
00000400:00000200:0.0:1788576092.566916:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576092.566922:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576092.566927:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576092.566931:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576092.566934:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576092.566937:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788576092.566941:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576092.566944:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788576092.566948:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576092.566950:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576092.566953:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576092.566956:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576092.566960:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576092.566962:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576092.566965:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576092.566968:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576117 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566977:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576092.566980:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576117 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.566986:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576092.566989:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576092.566992:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576092.566997:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576092.567000:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576092.567002:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307696167
00000100:00000001:0.0:1788576092.567005:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307697082
00000100:00000001:0.0:1788576092.567008:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307696167
00000100:00000001:0.0:1788576092.567010:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307697082
00000100:00000001:0.0:1788576092.567012:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576092.567014:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576092.567016:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576092.567020:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576092/real 1788576092]  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576092.567028:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576092.567029:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576092.567031:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576092.567034:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576092.567036:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576092.567038:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576092.567041:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576092.567048:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567060:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576092.567061:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567062:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576092.567064:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576092.567066:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567072:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576092.567075:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567081:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576092.567083:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567085:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576092.567086:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567088:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576092.567090:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576092.567093:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576092.567096:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576092.567098:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576092.567102:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576092.567108:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393258624:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576092.567112:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576092.567116:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567123:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576092.567124:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576092.567128:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576092.567130:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576092.567132:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576092.567135:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393258752:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576092.567138:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576092.567141:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576092.567142:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576092.567144:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576092.567147:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576092.567149:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576092.567152:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576092.567155:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576092.567158:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576092.567161:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393258752, portal 4
00000100:00000001:0.0:1788576092.567163:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576092.567164:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:0.0:1788576092.567170:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576117 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567176:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576092.567178:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576092.567180:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576092.567183:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393258752, offset 0
00000400:00000200:0.0:1788576092.567186:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576092.567190:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576092.567193:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576092.567196:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576092.567198:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576092.567200:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576092.567202:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576092.567205:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331277:331277:256:4294967295] 192.168.202.110@tcp LPNI seq info [331234:331234:8:4294967295]
00000400:00000200:0.0:1788576092.567209:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576092.567214:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576092.567219:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576092.567223:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576092.567226:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576092.567229:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1788576092.567233:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576092.567235:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1788576092.567239:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576092.567241:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576092.567245:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576092.567248:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576092.567252:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576092.567253:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576092.567256:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576092.567259:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576117 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567267:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576092.567269:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576117 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567275:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576092.567277:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576092.567280:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576092.567284:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576092.567287:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576092.567289:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307696313
00000100:00000001:0.0:1788576092.567291:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307697082
00000100:00000001:0.0:1788576092.567294:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307696313
00000100:00000001:0.0:1788576092.567296:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307697082
00000100:00000001:0.0:1788576092.567298:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576092.567300:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576092.567302:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576092.567307:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576092/real 1788576092]  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576092.567315:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576092.567317:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576092.567319:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576092.567321:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576092.567323:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576092.567325:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576092.567327:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576092.567336:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567342:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576092.567343:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567345:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576092.567347:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576092.567350:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567356:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576092.567360:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567366:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576092.567369:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567370:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576092.567372:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567373:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576092.567375:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576092.567378:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576092.567382:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576092.567384:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576092.567389:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576092.567424:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393258752:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576092.567429:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567431:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576092.567434:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e00700 x1875444393258624/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567440:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576092.567442:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576092.567444:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576092.567446:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576092.567448:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576092.567449:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576092.567451:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576092.567453:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:0.0:1788576092.567456:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576092.567457:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576092.567458:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e00700.
00000100:00000001:0.0:1788576092.567460:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576092.567462:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567463:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576092.567466:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393258752/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576117 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576092.567472:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576092.567473:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576092.567474:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576092.567476:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576092.567477:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576092.567478:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576092.567480:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576092.567482:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11800.
02000000:00000001:0.0:1788576092.567484:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576092.567485:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576092.567487:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:0.0:1788576092.567489:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576092.567490:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567492:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576092.567496:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576092.567497:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576092.567500:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576092.567501:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576092.951103:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576092.951110:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576092.951115:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576092.951116:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576092.951119:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576092.951120:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576092.951210:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576092.951214:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576092.951217:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576093.077570:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576093.077580:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576093.077583:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576093.077591:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576093.077596:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576093.077600:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576093.077604:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576093.077607:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576093.077611:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576093.525591:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307697082, at 4307697088
00000100:00000010:0.0:1788576093.525608:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000040:1.0:1788576093.525665:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307697082, at 4307697088
00000100:00000010:1.0:1788576093.525670:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7180.
00010000:00000001:3.0:1788576093.973506:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576093.973513:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576093.973519:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576093.973520:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576093.973524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576093.973525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576093.973716:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576093.973720:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576093.973723:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576094.102808:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576094.102814:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576094.102815:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576094.102819:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576094.102822:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576094.102824:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576094.102826:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576094.102827:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576094.102829:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576094.997957:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576094.997964:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576094.997969:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576094.997970:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576094.997973:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576094.997974:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576094.998636:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576094.998639:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576094.998643:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576095.125515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576095.125522:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576095.125524:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576095.125529:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576095.125533:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576095.125537:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576095.125540:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576095.125542:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576095.125545:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788576095.637650:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788576095.637658:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788576095.637663:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:1.0:1788576095.638027:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788576095.638034:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788576095.638045:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788576095.638049:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788576095.638051:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788576096.021971:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576096.021979:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576096.021985:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576096.021986:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576096.021990:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576096.021991:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576096.022101:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576096.022105:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576096.022108:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576096.152009:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576096.152015:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576096.152018:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576096.152024:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576096.152027:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576096.152031:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576096.152033:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576096.152036:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576096.152039:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576097.046762:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576097.046768:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576097.046773:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576097.046776:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576097.046779:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576097.046782:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576097.046784:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576097.046786:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576097.046788:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576097.046790:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576097.046794:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576097.046795:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576097.046796:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576097.046797:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576097.046798:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576097.046799:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576097.046801:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576097.046802:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576097.046894:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576097.046898:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576097.046901:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576097.173490:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576097.173498:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576097.173500:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576097.173505:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576097.173509:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576097.173512:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.173515:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576097.173517:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.173520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576097.685908:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576097.685919:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788576097.685922:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576097.685925:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576097.685930:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576097.685937:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13028
00000400:00000200:1.0:1788576097.685946:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576097.685952:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576097.685956:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576097.685959:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.685961:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576097.685963:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.685967:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576097.685970:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576097.685972:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788576097.685975:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576097.685978:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788576097.685981:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788576097.685983:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788576097.685986:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.685988:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576097.685990:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.685992:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576097.685995:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788576097.685998:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686001:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576097.686003:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576097.686006:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576097.686009:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576097.686012:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788576097.686014:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576097.686023:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5180.
00000020:00000040:1.0:1788576097.686027:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788576097.686031:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576097.686032:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576097.686035:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576097.686037:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576097.686041:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576097.686048:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788576097.686055:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576097.686064:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576097.686073:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576097.686078:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576097.686082:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d24a6de] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576097.686101:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576097.686104:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576097.686107:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788576097.686109:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576097.686110:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576097.686113:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576097.686116:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13028
00000400:00000200:1.0:1788576097.686120:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576097.686122:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576097.686124:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576097.686127:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686129:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576097.686130:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686132:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576097.686134:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576097.686135:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788576097.686136:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576097.686138:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788576097.686140:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788576097.686141:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788576097.686144:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686146:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576097.686147:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686149:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576097.686151:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788576097.686153:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686155:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576097.686157:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576097.686162:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576097.686164:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576097.686166:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576097.686170:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788576097.686172:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576097.686174:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576097.686178:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6a00.
00000020:00000040:1.0:1788576097.686180:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788576097.686182:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576097.686183:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576097.686185:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576097.686186:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576097.686188:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576097.686192:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788576097.686195:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576097.686199:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576097.686206:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576097.686208:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576097.686212:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d9902ce] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576097.686218:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576097.686221:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576097.686224:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788576097.686225:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576097.686226:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576097.686229:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576097.686231:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13028
00000400:00000200:1.0:1788576097.686235:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576097.686238:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576097.686240:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576097.686243:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686246:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576097.686247:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686249:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576097.686251:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576097.686252:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788576097.686254:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576097.686255:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788576097.686257:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788576097.686258:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788576097.686261:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686263:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576097.686264:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686266:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576097.686268:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788576097.686271:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576097.686272:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576097.686274:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576097.686277:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576097.686279:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576097.686281:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576097.686282:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788576097.686284:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576097.686286:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576097.686289:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4a80.
00000020:00000040:1.0:1788576097.686292:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788576097.686294:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576097.686295:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576097.686296:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576097.686298:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576097.686299:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576097.686302:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788576097.686306:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576097.686310:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576097.686317:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576097.686319:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576097.686322:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e0644662] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576097.686329:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576097.686332:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788576097.686335:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788576097.686340:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13038)
00000100:00000001:0.0:1788576097.686763:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576097.686768:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576097.686771:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576097.686778:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.686787:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576097.686789:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576097.686792:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576097.686794:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576097.686797:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576097.686802:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393258880:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576097.686807:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576097.686811:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576097.686814:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576097.686816:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576097.686822:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576097.686824:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576097.686830:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576097.686834:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576097.686840:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576097.686879:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393258880, portal 25
00000100:00000001:0.0:1788576097.686882:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576097.686884:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345470336 : -130483364081280 : ffff8953796f5180)
00000100:00000040:0.0:1788576097.686891:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.686898:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576097.686901:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576097.686905:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576097.686907:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393258880, offset 0
00000400:00000200:0.0:1788576097.686923:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576097.686931:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576097.686937:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576097.686941:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576097.686945:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576097.686949:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576097.686951:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576097.686954:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331278:331278:256:4294967295] 192.168.202.110@tcp LPNI seq info [331235:331235:8:4294967295]
00000400:00000200:0.0:1788576097.686959:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576097.686967:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576097.686972:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576097.686979:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576097.686986:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576097.686989:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788576097.686995:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576097.686998:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1788576097.687002:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576097.687005:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576097.687012:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576097.687017:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576097.687021:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576097.687023:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576097.687037:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576097.687042:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.687052:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576097.687055:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576122 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.687063:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576097.687064:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576097.687070:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576097.687074:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576097.687077:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307702202
00000100:00000010:0.0:1788576097.687081:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576097.687087:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576097.687092:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307702202
00000100:00000010:0.0:1788576097.687095:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7d20.
00000100:00000001:0.0:1788576097.687098:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576097.687101:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576097.687104:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576097.687106:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576097.687111:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576097/real 1788576097]  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576097.687124:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576097.687126:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576097.687127:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576097.687130:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576097.687131:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576097.687134:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576097.687137:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576097.687144:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.687151:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576097.687152:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.687154:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576097.687156:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576097.687161:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.687168:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576097.687172:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.687179:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576097.687182:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.687184:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576097.687185:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.687187:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576097.687189:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576097.687193:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576097.687197:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576097.687199:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576097.687204:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576097.687212:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393258880:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576097.687216:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576097.687220:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.687226:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576097.687227:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576097.687230:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576097.687231:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576097.687234:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576097.687236:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576097.687239:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576097.687241:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576097.687243:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576097.687245:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576097.687248:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576097.687249:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576097.687253:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576097.687256:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576097.687259:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576097.687262:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393259008, portal 10
00000100:00000001:0.0:1788576097.687264:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576097.687265:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345476608 : -130483364075008 : ffff8953796f6a00)
00000100:00000040:0.0:1788576097.687271:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.687277:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576097.687279:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576097.687281:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576097.687284:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393259008, offset 0
00000400:00000200:0.0:1788576097.687287:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576097.687291:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576097.687294:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576097.687297:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576097.687300:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576097.687302:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576097.687304:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576097.687306:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331279:331279:256:4294967295] 192.168.202.110@tcp LPNI seq info [331236:331236:8:4294967295]
00000400:00000200:0.0:1788576097.687310:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576097.687316:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576097.687320:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576097.687324:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576097.687327:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576097.687330:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788576097.687446:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576097.687449:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1788576097.687453:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576097.687455:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576097.687459:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576097.687462:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576097.687466:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576097.687467:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576097.687470:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576097.687474:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.687481:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576097.687484:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576122 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.687490:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576097.687492:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576097.687495:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576097.687498:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576097.687502:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576097.687504:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307701288
00000100:00000001:0.0:1788576097.687507:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307702202
00000100:00000001:0.0:1788576097.687509:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307701288
00000100:00000001:0.0:1788576097.687511:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307702202
00000100:00000001:0.0:1788576097.687513:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576097.687515:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576097.687517:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576097.687521:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576097/real 1788576097]  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576097.688086:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576097.688088:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576097.688090:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576097.688093:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576097.688094:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576097.688096:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576097.688101:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576097.688108:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.688115:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576097.688116:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.688118:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576097.688120:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576097.688123:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.688130:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576097.688135:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.688142:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576097.688144:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.688146:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576097.688147:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.688149:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576097.688151:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576097.688154:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576097.688157:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576097.688160:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576097.688164:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576097.688171:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576097.688175:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576097.688178:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.688185:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576097.688186:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576097.688189:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576097.688190:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576097.688192:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576097.688195:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576097.688198:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576097.688201:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576097.688202:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576097.688204:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576097.688208:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576097.688210:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576097.688214:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576097.688217:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576097.688220:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576097.688223:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393259136, portal 4
00000100:00000001:0.0:1788576097.688225:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576097.688226:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345468544 : -130483364083072 : ffff8953796f4a80)
00000100:00000040:0.0:1788576097.688232:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.688238:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576097.688240:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576097.688243:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576097.688245:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393259136, offset 0
00000400:00000200:0.0:1788576097.688249:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576097.688254:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576097.688258:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576097.688261:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576097.688264:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576097.688266:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576097.688268:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576097.688271:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331280:331280:256:4294967295] 192.168.202.110@tcp LPNI seq info [331237:331237:8:4294967295]
00000400:00000200:0.0:1788576097.688275:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576097.688281:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576097.688286:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576097.688289:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576097.688293:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576097.688296:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1788576097.688300:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576097.688302:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1788576097.688306:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576097.688308:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576097.688311:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576097.688314:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576097.688317:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576097.688319:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576097.688322:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576097.688325:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.689626:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576097.689632:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576122 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.689638:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576097.689640:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576097.689646:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576097.689650:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576097.689655:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576097.689657:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307701436
00000100:00000001:0.0:1788576097.689660:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307702202
00000100:00000001:0.0:1788576097.689662:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307701436
00000100:00000001:0.0:1788576097.689664:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307702202
00000100:00000001:0.0:1788576097.689666:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576097.689668:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576097.689670:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576097.689674:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576097/real 1788576097]  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576097.689682:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576097.689684:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576097.689685:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576097.689688:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576097.689690:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576097.689692:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576097.689695:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576097.689702:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.689708:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576097.689710:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.689711:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576097.689713:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576097.689716:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.689722:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576097.689726:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.689732:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576097.689734:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.689736:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576097.689737:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.689739:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576097.689741:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576097.689743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576097.689746:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576097.689749:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576097.689752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576097.689760:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576097.689764:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.689766:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576097.689769:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5180 x1875444393258880/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.689776:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576097.689779:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576097.689781:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576097.689785:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576097.689787:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576097.689789:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576097.689791:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576097.689794:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:0.0:1788576097.689797:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576097.689799:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576097.689801:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5180.
00000100:00000001:0.0:1788576097.689804:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576097.689805:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.689806:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576097.689810:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6a00 x1875444393259008/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.689816:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576097.689817:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576097.689819:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576097.689821:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576097.689823:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576097.689824:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576097.689826:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576097.689827:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:0.0:1788576097.689829:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576097.689830:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576097.689832:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6a00.
00000100:00000001:0.0:1788576097.689834:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576097.689835:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.689836:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576097.689840:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4a80 x1875444393259136/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576122 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576097.689856:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576097.689858:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576097.689859:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576097.689862:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576097.689863:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576097.689864:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576097.689866:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576097.689868:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:0.0:1788576097.689870:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576097.689871:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576097.689873:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4a80.
00000100:00000001:0.0:1788576097.689875:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576097.689876:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.689878:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576097.689886:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576097.689888:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576097.689890:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576097.689892:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576098.069507:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576098.069514:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576098.069520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576098.069521:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576098.069524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576098.069525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576098.070058:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576098.070062:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576098.070065:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576098.197509:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576098.197517:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576098.197519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576098.197524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576098.197527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576098.197531:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576098.197534:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576098.197536:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576098.197539:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576098.645485:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307702202, at 4307702208
00000100:00000010:1.0:1788576098.645500:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7d20.
00000100:00000040:0.0:1788576098.645523:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307702202, at 4307702208
00000100:00000010:0.0:1788576098.645530:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00010000:00000001:3.0:1788576099.093558:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576099.093567:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576099.093580:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576099.093583:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576099.093588:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576099.093590:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576099.093592:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576099.093595:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576099.093606:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576099.093608:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576099.093611:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576099.093612:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576099.095698:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576099.095702:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576099.095705:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576099.221504:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576099.221511:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576099.221513:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576099.221519:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576099.221523:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576099.221526:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576099.221528:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576099.221531:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576099.222073:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576100.117582:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576100.117590:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576100.117595:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576100.117596:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576100.117599:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576100.117601:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576100.117727:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576100.117731:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576100.117735:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576100.245671:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576100.245679:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576100.245682:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576100.245689:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576100.245693:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576100.245698:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576100.245701:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576100.245704:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576100.245707:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576101.141512:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576101.141519:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576101.141524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576101.141525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576101.141529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576101.141530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576101.142381:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576101.142385:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576101.142389:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576101.269912:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576101.269919:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576101.269921:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576101.269927:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576101.269931:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576101.269934:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576101.269937:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576101.269939:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576101.269942:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576102.166086:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576102.166094:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576102.166099:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576102.166100:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576102.166103:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576102.166104:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576102.166741:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576102.166745:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576102.167107:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576102.293552:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576102.293560:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576102.293562:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576102.293568:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576102.293571:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576102.293574:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.293577:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576102.293579:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.293582:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576102.805543:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576102.805553:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788576102.805556:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576102.805559:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576102.805564:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576102.805571:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13033
00000400:00000200:1.0:1788576102.805581:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576102.805586:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576102.805589:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576102.805592:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805595:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576102.805598:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805600:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576102.805604:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576102.805606:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788576102.805609:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576102.805612:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788576102.805614:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788576102.805617:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788576102.805620:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805622:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576102.805624:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805625:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576102.805628:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788576102.805631:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805633:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576102.805636:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576102.805639:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576102.805642:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.805644:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788576102.805647:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576102.805657:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4380.
00000020:00000040:1.0:1788576102.805660:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788576102.805664:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576102.805666:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.805669:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576102.805671:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576102.805676:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576102.805684:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758000.
00000100:00000001:1.0:1788576102.805691:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576102.805712:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576102.805722:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576102.805727:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576102.805732:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000579365b7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576102.805760:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576102.805763:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576102.805767:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788576102.805769:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576102.805770:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576102.805773:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576102.805776:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13033
00000400:00000200:1.0:1788576102.805780:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576102.805782:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576102.805785:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576102.805788:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805790:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576102.805792:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805794:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576102.805796:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576102.805797:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788576102.805798:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576102.805800:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788576102.805802:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788576102.805804:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788576102.805806:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805809:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576102.805810:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805812:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576102.805814:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788576102.805816:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576102.805818:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576102.805820:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576102.805826:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576102.805828:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.805830:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576102.805843:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788576102.805847:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.805848:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576102.805852:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4000.
00000020:00000040:1.0:1788576102.805864:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788576102.805867:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576102.805869:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.805870:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576102.805871:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576102.805873:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576102.805878:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788576102.805881:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576102.805887:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576102.805891:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576102.805896:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576102.805898:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576102.805900:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1788576102.805903:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576102.805910:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788576102.805911:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004e36d741] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.805927:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576102.805929:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576102.805932:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576102.805936:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576102.805941:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.805941:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576102.805948:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1788576102.805950:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393259264:192.168.202.110@tcp:250:kworker.0
00000100:00080000:1.0:1788576102.805956:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576102.805957:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1788576102.805959:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576102.805962:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:0.0:1788576102.805963:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576102.805967:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.805967:0:186668:0:(import.c:520:import_select_connection()) Process entered
02000000:00000001:0.0:1788576102.805971:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:1.0:1788576102.805974:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13033
02000000:00000010:0.0:1788576102.805979:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
00000400:00000200:1.0:1788576102.805980:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1788576102.805984:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1788576102.805987:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576102.805991:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1788576102.805993:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788576102.805997:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000400:00000010:0.0:1788576102.805999:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000100:00000001:1.0:1788576102.806001:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576102.806005:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000400:00000010:0.0:1788576102.806007:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000001:1.0:1788576102.806008:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576102.806013:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576102.806014:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000200:0.0:1788576102.806015:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393259264, portal 25
00000020:00000001:1.0:1788576102.806016:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:0.0:1788576102.806020:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000020:00000040:1.0:1788576102.806021:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000001:0.0:1788576102.806024:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345466752 : -130483364084864 : ffff8953796f4380)
00000020:00000001:1.0:1788576102.806025:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788576102.806030:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788576102.806034:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1788576102.806036:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576102.806040:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576102.806045:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576102.806049:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1788576102.806050:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:0.0:1788576102.806055:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:1.0:1788576102.806055:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000400:00000010:0.0:1788576102.806061:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00080000:1.0:1788576102.806062:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:0.0:1788576102.806065:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393259264, offset 0
00000100:00000001:1.0:1788576102.806066:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576102.806071:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576102.806074:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1788576102.806075:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:1.0:1788576102.806079:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.806082:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576102.806084:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:0.0:1788576102.806087:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:1.0:1788576102.806088:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.806092:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576102.806095:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000010:1.0:1788576102.806097:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5f80.
00000020:00000040:1.0:1788576102.806101:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000200:0.0:1788576102.806102:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:1.0:1788576102.806107:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1788576102.806110:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:1.0:1788576102.806111:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576102.806114:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1788576102.806117:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1788576102.806117:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576102.806120:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576102.806123:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000010:1.0:1788576102.806127:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000400:00000200:0.0:1788576102.806129:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331281:331281:256:4294967295] 192.168.202.110@tcp LPNI seq info [331238:331238:8:4294967295]
00000100:00000001:1.0:1788576102.806133:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576102.806137:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:1.0:1788576102.806142:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788576102.806153:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1788576102.806159:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576102.806163:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788576102.806164:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1788576102.806170:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d5187ab7] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:0.0:1788576102.806172:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576102.806182:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1788576102.806182:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576102.806186:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.0:1788576102.806188:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000040:1.0:1788576102.806190:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:0.1:1788576102.806196:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1788576102.806198:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13044)
00000800:00000200:0.1:1788576102.806200:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788576102.806206:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576102.806208:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576102.806215:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576102.806221:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576102.806226:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576102.806228:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576102.806233:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576102.806237:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806246:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576102.806250:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576127 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806257:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576102.806259:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576102.806264:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576102.806268:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576102.806270:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307707321
00000100:00000010:0.0:1788576102.806275:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576102.806281:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576102.806286:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307707321
00000100:00000010:0.0:1788576102.806289:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7b40.
00000100:00000001:0.0:1788576102.806292:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576102.806294:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576102.806297:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576102.806299:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576102.806304:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576102/real 1788576102]  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576102.806313:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576102.806314:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576102.806316:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576102.806319:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576102.806321:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576102.806323:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576102.806326:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576102.806340:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806347:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576102.806349:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806351:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576102.806353:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576102.806357:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806363:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576102.806368:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806376:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576102.806379:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806380:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576102.806381:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806384:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576102.806385:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576102.806388:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576102.806416:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576102.806419:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576102.806436:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576102.806445:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393259264:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576102.806449:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806451:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576102.806454:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4380 x1875444393259264/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806466:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576102.806468:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576102.806471:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576102.806475:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576102.806476:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576102.806478:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576102.806480:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576102.806482:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758000.
02000000:00000001:0.0:1788576102.806485:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576102.806487:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576102.806488:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4380.
00000100:00000001:0.0:1788576102.806491:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576102.806492:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806494:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806501:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576102.806503:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576102.806505:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576102.806506:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576102.806508:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576102.806512:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806518:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576102.806520:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576102.806522:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576102.806523:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576102.806525:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576102.806528:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259392:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576102.806532:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576102.806534:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576102.806535:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576102.806537:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576102.806540:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576102.806541:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576102.806545:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576102.806548:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576102.806551:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576102.806555:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393259392, portal 10
00000100:00000001:0.0:1788576102.806556:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576102.806558:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345465856 : -130483364085760 : ffff8953796f4000)
00000100:00000040:0.0:1788576102.806564:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806570:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576102.806573:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576102.806575:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576102.806577:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393259392, offset 0
00000400:00000200:0.0:1788576102.806581:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576102.806585:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576102.806589:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576102.806592:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576102.806594:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576102.806596:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576102.806599:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576102.806602:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331282:331282:256:4294967295] 192.168.202.110@tcp LPNI seq info [331239:331239:8:4294967295]
00000400:00000200:0.0:1788576102.806606:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576102.806612:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576102.806616:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576102.806619:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576102.806623:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576102.806625:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788576102.806629:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576102.806632:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788576102.806636:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576102.806637:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576102.806640:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576102.806644:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576102.806646:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576102.806648:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576102.806651:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576102.806654:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806662:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576102.806664:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576127 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806670:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576102.806671:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576102.806674:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576102.806678:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576102.806681:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576102.806683:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307706407
00000100:00000001:0.0:1788576102.806686:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307707321
00000100:00000001:0.0:1788576102.806688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307706407
00000100:00000001:0.0:1788576102.806691:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307707321
00000100:00000001:0.0:1788576102.806693:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576102.806695:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576102.806696:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576102.806700:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576102/real 1788576102]  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576102.806707:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576102.806709:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576102.806710:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576102.806713:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576102.806714:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576102.806716:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576102.806719:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576102.806727:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806739:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576102.806741:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806742:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576102.806743:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576102.806746:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806750:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576102.806752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806756:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576102.806758:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806758:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576102.806759:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806760:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576102.806761:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576102.806763:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576102.806765:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576102.806766:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576102.806768:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576102.806773:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259392:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576102.806775:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576102.806777:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806781:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576102.806782:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576102.806784:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576102.806785:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576102.806786:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576102.806788:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259520:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576102.806790:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576102.806791:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576102.806792:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576102.806793:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576102.806795:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576102.806796:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576102.806799:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576102.806801:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576102.806803:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576102.806805:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393259520, portal 4
00000100:00000001:0.0:1788576102.806806:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576102.806807:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345473920 : -130483364077696 : ffff8953796f5f80)
00000100:00000040:0.0:1788576102.806811:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806814:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576102.806816:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576102.806817:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576102.806819:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393259520, offset 0
00000400:00000200:0.0:1788576102.806821:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576102.806823:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576102.806825:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576102.806827:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576102.806829:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576102.806830:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576102.806832:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576102.806834:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331283:331283:256:4294967295] 192.168.202.110@tcp LPNI seq info [331240:331240:8:4294967295]
00000400:00000200:0.0:1788576102.806836:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576102.806841:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576102.806844:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576102.806846:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576102.806848:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576102.806850:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1788576102.806852:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576102.806854:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1788576102.806867:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576102.806869:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:0.0:1788576102.806871:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576102.806873:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576102.806875:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:0.0:1788576102.806876:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:0.0:1788576102.806878:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576102.806880:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806885:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576102.806886:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576127 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806890:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576102.806891:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576102.806893:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576102.806895:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576102.806897:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576102.806898:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307706553
00000100:00000001:0.0:1788576102.806900:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307707321
00000100:00000001:0.0:1788576102.806901:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307706553
00000100:00000001:0.0:1788576102.806902:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307707321
00000100:00000001:0.0:1788576102.806903:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576102.806904:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576102.806905:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576102.806907:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576102/real 1788576102]  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576102.806912:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576102.806913:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576102.806914:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576102.806915:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576102.806916:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576102.806917:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576102.806919:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576102.806923:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806927:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576102.806928:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806929:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576102.806930:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576102.806931:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806935:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576102.806937:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806940:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576102.806942:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806942:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576102.806943:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806944:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576102.806945:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576102.806946:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576102.806948:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576102.806949:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576102.806953:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576102.806960:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259520:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576102.806964:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806965:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576102.806968:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4000 x1875444393259392/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.806974:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576102.806975:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576102.806977:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576102.806979:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576102.806981:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576102.806982:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576102.806984:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576102.806986:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:0.0:1788576102.806988:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576102.806989:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576102.806990:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4000.
00000100:00000001:0.0:1788576102.806993:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576102.806994:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.806996:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576102.806998:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5f80 x1875444393259520/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576127 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576102.807002:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576102.807003:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576102.807004:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576102.807005:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576102.807006:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576102.807007:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576102.807008:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576102.807008:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:0.0:1788576102.807009:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576102.807010:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576102.807012:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5f80.
00000100:00000001:0.0:1788576102.807013:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576102.807014:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.807015:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576102.807018:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576102.807019:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576102.807020:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576102.807021:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576103.189565:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576103.189572:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576103.189578:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576103.189580:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576103.189583:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576103.189584:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576103.189719:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576103.189723:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576103.189726:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576103.317517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576103.317525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576103.317527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576103.317541:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576103.317544:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576103.317551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576103.317554:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576103.317556:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576103.317559:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576103.765519:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307707321, at 4307707328
00000100:00000010:0.0:1788576103.765534:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000040:1.0:1788576103.765947:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307707321, at 4307707328
00000100:00000010:1.0:1788576103.765955:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7b40.
00010000:00000001:3.0:1788576104.214179:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576104.214185:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576104.214189:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576104.214190:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576104.214192:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576104.214193:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576104.214491:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576104.214495:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576104.214499:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576104.341570:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576104.341577:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576104.341587:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576104.341594:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576104.341598:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576104.341605:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576104.341608:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576104.341613:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576104.341616:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576105.237539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576105.237546:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576105.237551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576105.237552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576105.237555:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576105.237557:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576105.237706:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576105.237709:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576105.237712:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576105.365496:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576105.365502:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576105.365504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576105.365508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576105.365511:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576105.365514:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576105.365517:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576105.365518:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576105.365520:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576106.261538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576106.261545:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576106.261561:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576106.261562:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576106.261565:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576106.261567:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576106.261716:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576106.261720:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576106.261724:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576106.389529:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576106.389538:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576106.389540:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576106.389545:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576106.389549:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576106.389553:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576106.389556:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576106.389558:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576106.389561:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576107.285467:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576107.285475:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576107.285481:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576107.285484:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576107.285487:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576107.285490:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576107.285493:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576107.285495:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576107.285496:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576107.285498:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576107.285500:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576107.285501:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576107.285502:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576107.285503:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576107.285504:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576107.285504:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576107.285505:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576107.285507:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576107.288188:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576107.288196:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576107.288201:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576107.413534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576107.413543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576107.413545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576107.413551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576107.413556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576107.413560:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576107.413562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576107.413565:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576107.413569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576107.925640:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576107.925651:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788576107.925653:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576107.925657:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576107.925661:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576107.925667:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13039
00000400:00000200:1.0:1788576107.925676:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576107.925682:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576107.925684:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576107.925687:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.925690:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576107.925692:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.925695:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576107.925698:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576107.925700:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788576107.925703:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576107.925705:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788576107.925708:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788576107.925710:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788576107.925713:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.925714:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576107.925716:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.925718:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576107.925720:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788576107.925723:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576107.925726:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576107.925728:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576107.925731:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576107.925734:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576107.925735:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788576107.925738:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576107.925746:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4700.
00000020:00000040:1.0:1788576107.925749:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788576107.925753:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576107.925754:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576107.925758:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576107.925760:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576107.925763:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576107.925770:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788576107.925775:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576107.925783:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576107.925793:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576107.925797:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576107.925801:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1b312c] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576107.925825:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576107.925828:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576107.925831:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788576107.925832:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576107.925834:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576107.925837:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576107.925839:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13039
00000400:00000200:1.0:1788576107.925843:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576107.925845:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576107.925847:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576107.925850:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.925852:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576107.925854:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.925856:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576107.925857:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576107.925859:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788576107.925860:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576107.925862:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788576107.925864:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788576107.925866:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788576107.925897:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.925899:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576107.925901:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.925903:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788576107.925904:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576107.925910:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576107.925913:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576107.925921:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576107.925930:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576107.925932:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576107.925935:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576107.925938:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576107.925940:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576107.925944:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393259648:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576107.925947:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576107.925950:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576107.925953:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576107.925954:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576107.925958:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576107.925960:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576107.925964:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576107.925967:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576107.925971:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576107.925976:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393259648, portal 25
00000100:00000001:0.0:1788576107.925978:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576107.925979:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345467648 : -130483364083968 : ffff8953796f4700)
00000100:00000040:0.0:1788576107.925983:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576107.925988:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576107.925990:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576107.925992:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4550.
00000100:00000200:0.0:1788576107.925993:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393259648, offset 0
00000400:00000200:0.0:1788576107.925997:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576107.926003:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576107.926007:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576107.926011:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576107.926014:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576107.926017:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576107.926018:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576107.926021:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331284:331284:256:4294967295] 192.168.202.110@tcp LPNI seq info [331241:331241:8:4294967295]
00000400:00000200:0.0:1788576107.926024:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576107.926030:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576107.926034:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576107.926038:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9c00.
00000800:00000200:0.0:1788576107.926042:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576107.926045:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788576107.926065:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576107.926068:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576107.926070:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307712441
00000100:00000010:0.0:1788576107.926073:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576107.926079:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576107.926083:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307712441
00000100:00000010:0.0:1788576107.926085:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7ba0.
00000100:00000001:0.0:1788576107.926087:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576107.926089:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576107.926091:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.926094:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.926095:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576107.926101:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576107.926102:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788576107.926104:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576107.926105:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576107.926106:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.926107:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788576107.928298:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788576107.928320:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576107.928323:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576107.928327:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576107.928335:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576107.928337:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576107.928341:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576107.928345:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788576107.928348:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576107.928350:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576107.928359:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7800.
00000020:00000040:1.0:1788576107.928363:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788576107.928368:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576107.928369:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576107.928373:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576107.928375:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576107.928379:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576107.928386:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788576107.928407:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576107.928417:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576107.928427:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576107.928431:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576107.928435:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004b023ad3] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576107.928466:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576107.928470:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576107.928474:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788576107.928476:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576107.928478:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576107.928480:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576107.928484:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13039
00000400:00000200:1.0:1788576107.928491:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576107.928497:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576107.928500:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576107.928503:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.928506:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576107.928509:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.928511:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576107.928514:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576107.928516:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788576107.928518:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576107.928520:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788576107.928522:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788576107.928524:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:0.0:1788576107.928528:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1788576107.928528:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576107.928533:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576107.928533:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1788576107.928537:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1788576107.928537:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576107.928542:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000040:0.0:1788576107.928546:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1788576107.928547:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788576107.928553:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576107.928558:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576107.928559:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.928561:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576107.928564:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576107.928565:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576107.928569:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576107.928570:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576107.928571:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576107.928576:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576107.928577:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259776:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1788576107.928582:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576107.928583:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576107.928588:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:1.0:1788576107.928588:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
02000000:00000001:0.0:1788576107.928590:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1788576107.928593:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576107.928595:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1788576107.928599:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576107.928602:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5800.
00000100:00000010:1.0:1788576107.928605:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7480.
02000000:00000001:0.0:1788576107.928606:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1788576107.928611:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788576107.928615:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1788576107.928616:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1788576107.928618:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576107.928622:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488da580.
00000100:00000001:1.0:1788576107.928622:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576107.928624:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576107.928628:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576107.928632:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
02000000:00000010:1.0:1788576107.928633:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a000.
00000100:00000001:1.0:1788576107.928639:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1788576107.928641:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393259776, portal 10
00000100:00000001:0.0:1788576107.928646:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00100000:1.0:1788576107.928647:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576107.928649:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345480192 : -130483364071424 : ffff8953796f7800)
00000100:00000001:1.0:1788576107.928660:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1788576107.928662:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576107.928665:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.928673:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576107.928677:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576107.928679:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d46e8.
00000100:00000040:1.0:1788576107.928681:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e024d5a] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576107.928683:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393259776, offset 0
00000400:00000200:0.0:1788576107.928694:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1788576107.928698:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576107.928702:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788576107.928704:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1788576107.928710:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:0.0:1788576107.928714:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:1.0:1788576107.928717:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13049)
00000400:00000200:0.0:1788576107.928720:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576107.928725:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576107.928728:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576107.928731:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576107.928734:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331285:331285:255:4294967295] 192.168.202.110@tcp LPNI seq info [331242:331242:7:4294967295]
00000400:00000200:0.0:1788576107.928738:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576107.928746:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576107.928751:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576107.928757:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9400.
00000800:00000200:0.0:1788576107.928763:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576107.928767:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788576107.928772:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576107.928776:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576107.928779:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576107.928782:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307711529
00000100:00000001:0.0:1788576107.928785:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307712441
00000100:00000001:0.0:1788576107.928786:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307711529
00000100:00000001:0.0:1788576107.928788:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307712441
00000100:00000001:0.0:1788576107.928790:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576107.928791:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.928794:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.928795:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576107.928801:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576107.928802:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788576107.928804:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576107.928805:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576107.928806:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576107.928809:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576107.928813:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576107.928814:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576107.928816:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576107.928817:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576107.928818:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576107.928820:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576107.928822:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576107.928824:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576107.928825:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576107.928826:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576107.928828:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576107.928829:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576107.928831:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576107.928833:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488dae18.
00000400:00000010:0.0:1788576107.928836:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4ee0.
00000100:00000200:0.0:1788576107.928837:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393259904, portal 4
00000100:00000001:0.0:1788576107.928839:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576107.928839:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345479296 : -130483364072320 : ffff8953796f7480)
00000100:00000040:0.0:1788576107.928843:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576107.928848:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576107.928849:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576107.928851:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4d48.
00000100:00000200:0.0:1788576107.928852:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393259904, offset 0
00000400:00000200:0.0:1788576107.928855:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576107.928857:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576107.928860:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576107.928862:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576107.928864:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576107.928866:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576107.928879:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576107.928883:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331286:331286:254:4294967295] 192.168.202.110@tcp LPNI seq info [331243:331243:6:4294967295]
00000400:00000200:0.0:1788576107.928887:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576107.928892:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576107.928897:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576107.928900:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9900.
00000800:00000200:0.0:1788576107.928903:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576107.928907:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1788576107.928909:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576107.928911:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576107.928914:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576107.928916:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307711675
00000100:00000001:0.0:1788576107.928918:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307712441
00000100:00000001:0.0:1788576107.928920:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307711675
00000100:00000001:0.0:1788576107.928922:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307712441
00000100:00000001:0.0:1788576107.928923:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576107.928925:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.928927:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.928929:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576107.928931:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576107.928932:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1788576107.928934:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576107.928935:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576107.928937:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576107.928939:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576108.309519:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576108.309525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576108.309530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576108.309531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576108.309534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576108.309535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576108.309921:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576108.309925:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576108.309929:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576108.437564:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576108.437572:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576108.437575:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576108.437580:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576108.437583:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576108.437587:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576108.437589:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576108.437592:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576108.437595:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576108.885525:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307712441, at 4307712448
00000100:00000010:0.0:1788576108.885541:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00000100:00000040:1.0:1788576108.885635:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307712441, at 4307712448
00000100:00000010:1.0:1788576108.885641:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7ba0.
00010000:00000001:3.0:1788576109.333518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576109.333527:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576109.333529:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576109.333532:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576109.333537:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576109.333539:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576109.333541:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576109.333543:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576109.333546:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576109.333548:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576109.333551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576109.333552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576109.333761:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576109.333765:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576109.333768:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576109.461518:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576109.461525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576109.461527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576109.461532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576109.461536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576109.461540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576109.461542:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576109.461545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576109.461548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576110.359520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576110.359527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576110.359533:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576110.359535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576110.359538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576110.359540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576110.359645:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576110.359649:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576110.359652:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576110.487038:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576110.487047:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576110.487049:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576110.487055:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576110.487059:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576110.487063:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576110.487066:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576110.487068:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576110.487070:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1788576110.997705:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:2.0:1788576110.997750:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1788576110.997760:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576110.997764:0:2352:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:2.0:1788576110.997769:0:2352:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576110.997773:0:2352:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1788576110.997778:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1788576110.997781:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9c00.
00000400:00000200:2.0:1788576110.997787:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1788576110.997792:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1788576110.997799:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4550
00000400:00000010:2.0:1788576110.997802:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4550.
00000100:00000001:2.0:1788576110.997807:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576110.997817:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576110.997839:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576110.997844:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576132 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576110.997850:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576110.997852:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:2.0:1788576110.997861:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1788576110.997864:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9400.
00000400:00000200:2.0:1788576110.997867:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1788576110.997870:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1788576110.997873:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d46e8
00000400:00000010:2.0:1788576110.997875:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d46e8.
00000100:00000001:2.0:1788576110.997888:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576110.997892:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576110.997899:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576110.997901:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576132 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576110.997908:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576110.997909:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000800:00000100:2.0:1788576110.997914:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1788576110.997916:0:2352:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9900.
00000400:00000200:2.0:1788576110.997919:0:2352:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1788576110.997922:0:2352:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1788576110.997925:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4d48
00000400:00000010:2.0:1788576110.997926:0:2352:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4d48.
00000100:00000001:2.0:1788576110.997928:0:2352:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576110.997932:0:2352:0:(events.c:40:request_out_callback()) @@@ type 5, status -113  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576110.997938:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576110.997941:0:2352:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576132 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576110.997947:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576110.997949:0:2352:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576110.999725:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576110.999733:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576110.999735:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576110.999749:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576107/real 1788576110]  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576110.999765:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576110.999767:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576110.999771:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488dae18.
00000400:00000200:0.0:1788576110.999775:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4ee0
00000400:00000010:0.0:1788576110.999777:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4ee0.
00000100:00000001:0.0:1788576110.999780:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576110.999784:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576110.999792:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576110.999799:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576110.999800:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576110.999803:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576110.999805:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576110.999809:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576110.999815:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576110.999820:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576110.999827:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576110.999830:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576110.999831:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576110.999832:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576110.999835:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576110.999837:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576110.999841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576110.999845:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576110.999848:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576110.999852:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576110.999861:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576110.999865:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576110.999869:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576107/real 1788576110]  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576110.999888:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576110.999890:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576110.999891:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488da580.
00000400:00000200:0.0:1788576110.999894:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576110.999896:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576110.999898:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576110.999901:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576110.999909:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576110.999915:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576110.999916:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576110.999918:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576110.999919:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576110.999923:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576110.999928:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576110.999931:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576110.999937:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576110.999939:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576110.999941:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576110.999942:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576110.999944:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576110.999946:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576110.999948:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576110.999951:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576110.999953:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576110.999957:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576110.999963:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393259776:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576110.999967:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576110.999971:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576107/real 1788576110]  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576110.999978:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576110.999979:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576110.999981:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576110.999984:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576110.999985:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576110.999987:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576110.999990:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576110.999997:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576111.000003:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576111.000004:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.000006:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576111.000007:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576111.000010:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576111.000022:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576111.000025:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576111.000031:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576111.000033:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.000035:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576111.000036:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.000037:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576111.000038:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576111.000041:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576111.000044:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576111.000046:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576111.000050:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576111.000058:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393259648:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576111.000062:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.000064:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576111.000068:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7480 x1875444393259904/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576111.000074:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576111.000077:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576111.000081:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5000.
02000000:00000001:0.0:1788576111.000085:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576111.000088:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576111.000090:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576111.000092:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576111.000095:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a000.
02000000:00000001:0.0:1788576111.000098:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576111.000101:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576111.000103:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7480.
00000100:00000001:0.0:1788576111.000107:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576111.000108:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.000110:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576111.000113:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7800 x1875444393259776/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576111.000120:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576111.000121:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576111.000123:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd5800.
02000000:00000001:0.0:1788576111.000125:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576111.000126:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576111.000127:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576111.000129:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576111.000130:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:0.0:1788576111.000132:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576111.000133:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576111.000135:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7800.
00000100:00000001:0.0:1788576111.000137:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576111.000138:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.000139:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576111.000142:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4700 x1875444393259648/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576132 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576111.000148:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576111.000149:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576111.000151:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576111.000153:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576111.000154:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576111.000155:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576111.000157:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576111.000158:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:0.0:1788576111.000159:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576111.000160:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576111.000161:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4700.
00000100:00000001:0.0:1788576111.000164:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576111.000165:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.000167:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.000175:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576111.000177:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576111.000179:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576111.000180:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576111.381514:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576111.381532:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576111.381538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576111.381539:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576111.381543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576111.381544:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576111.381701:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576111.381705:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576111.381708:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576111.509523:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576111.509530:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576111.509533:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576111.509539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576111.509542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576111.509546:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.509549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576111.509552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576111.509555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576112.405510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576112.405517:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576112.405521:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576112.405522:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576112.405525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576112.405526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576112.405936:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576112.405940:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576112.405943:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576112.533525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576112.533531:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576112.533534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576112.533539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576112.533543:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576112.533547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576112.533550:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576112.533552:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576112.533555:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576113.046667:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576113.046677:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576113.046679:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576113.046684:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576113.046688:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576113.046695:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13044
00000400:00000200:3.0:1788576113.046704:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576113.046709:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576113.046712:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576113.046715:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576113.046718:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576113.046720:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576113.046723:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576113.046727:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576113.046729:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576113.046732:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576113.046735:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576113.046737:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576113.046740:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576113.046743:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576113.046745:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576113.046747:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576113.046749:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576113.046751:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576113.046755:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576113.046757:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576113.046759:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576113.046763:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576113.046766:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576113.046768:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576113.046770:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576113.046779:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0e00.
00000020:00000040:3.0:1788576113.046783:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576113.046786:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576113.046788:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576113.046791:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576113.046794:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576113.046798:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576113.046805:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
00000100:00000001:3.0:1788576113.046812:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576113.046821:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576113.046831:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576113.046835:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576113.046839:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fe0ae9fd] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576113.046865:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576113.046868:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576113.046872:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576113.046873:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576113.046875:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576113.046879:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576113.046913:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13044
00000100:00000001:0.0:1788576113.046915:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:3.0:1788576113.046919:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576113.046922:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576113.046926:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000400:00000200:3.0:1788576113.046926:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576113.046930:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576113.046936:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788576113.046937:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576113.046943:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576113.046948:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576113.046951:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576113.046953:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788576113.046954:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1788576113.046958:0:188751:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:0.0:1788576113.046960:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1788576113.046962:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
02000000:00000001:0.0:1788576113.046964:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:3.0:1788576113.046965:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:0.0:1788576113.046967:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1788576113.046970:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000100:00100000:0.0:1788576113.046974:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393260032:192.168.202.110@tcp:250:kworker.0
00000020:00000001:3.0:1788576113.046975:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576113.046981:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000001:0.0:1788576113.046983:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576113.046988:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:3.0:1788576113.046988:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
02000000:00000001:0.0:1788576113.046992:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576113.046993:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1788576113.046997:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:3.0:1788576113.046998:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576113.047002:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
02000000:00000010:0.0:1788576113.047003:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
00000020:00000040:3.0:1788576113.047005:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
02000000:00000001:0.0:1788576113.047008:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576113.047011:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576113.047016:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576113.047019:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1788576113.047019:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:0.0:1788576113.047024:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
02000000:00000001:3.0:1788576113.047028:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576113.047032:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1788576113.047033:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000001:3.0:1788576113.047034:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576113.047040:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000100:00000200:0.0:1788576113.047041:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393260032, portal 25
00000100:00000001:0.0:1788576113.047046:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000008:00000001:3.0:1788576113.047047:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576113.047049:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138818048 : -130483570733568 : ffff89536d1e0e00)
00000100:00000001:3.0:1788576113.047051:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576113.047056:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1788576113.047058:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e3480.
00000020:00000040:3.0:1788576113.047063:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:0.0:1788576113.047068:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1788576113.047069:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576113.047072:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576113.047073:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576113.047077:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:0.0:1788576113.047079:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
02000000:00000001:3.0:1788576113.047080:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:0.0:1788576113.047084:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393260032, offset 0
02000000:00000001:3.0:1788576113.047084:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576113.047093:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa000.
00000400:00000200:0.0:1788576113.047094:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576113.047099:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576113.047106:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1788576113.047107:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788576113.047116:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788576113.047118:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576113.047122:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576113.047123:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1788576113.047129:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000565d0ac4] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1788576113.047132:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576113.047139:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576113.047145:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1788576113.047147:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576113.047152:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331287:331287:256:4294967295] 192.168.202.110@tcp LPNI seq info [331244:331244:8:4294967295]
00000400:00000200:0.0:1788576113.047158:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576113.047169:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576113.047175:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576113.047180:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788576113.047187:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576113.047191:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788576113.047196:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576113.047199:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788576113.047203:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576113.047206:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788576113.047212:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576113.047216:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1788576113.047221:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1788576113.047222:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576113.047226:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00080000:3.0:1788576113.047228:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788576113.047231:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1788576113.047233:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00080000:3.0:1788576113.047234:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576113.047239:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00000200:0.0:1788576113.047241:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1788576113.047244:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13044
00000400:00000200:3.0:1788576113.047254:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576113.047260:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000400:00000200:3.0:1788576113.047260:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576113.047264:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576113.047267:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1788576113.047270:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576113.047275:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576113.047279:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576113.047282:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576113.047283:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:0.0:1788576113.047285:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000020:00000001:3.0:1788576113.047289:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576113.047292:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000100:00000001:0.0:1788576113.047293:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1788576113.047295:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576113.047299:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000400:00000200:0.0:1788576113.047300:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1788576113.047304:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:0.0:1788576113.047306:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307717562
00000020:00000001:3.0:1788576113.047309:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000010:0.0:1788576113.047313:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000040:3.0:1788576113.047314:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576113.047318:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576113.047321:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576113.047322:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788576113.047323:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576113.047328:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:0.0:1788576113.047330:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307717562
00000100:00080000:3.0:1788576113.047331:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000010:0.0:1788576113.047335:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d76c0.
00000100:00000001:3.0:1788576113.047336:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576113.047340:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1788576113.047341:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:3.0:1788576113.047344:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576113.047346:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576113.047349:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576113.047351:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576113.047353:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576113.047355:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000008:00100000:3.0:1788576113.047358:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000100:00100000:0.0:1788576113.047362:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576113/real 1788576113]  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:3.0:1788576113.047363:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576113.047368:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576113.047375:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000400:00000001:0.0:1788576113.047378:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:3.0:1788576113.047379:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000400:00000001:0.0:1788576113.047382:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:3.0:1788576113.047384:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:0.0:1788576113.047386:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
02000000:00000001:3.0:1788576113.047387:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576113.048007:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576113.048011:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576113.048016:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576113.048022:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576113.048032:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.048038:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576113.048041:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.048043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576113.048045:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576113.048050:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.048057:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576113.048062:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.048069:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576113.048072:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.048073:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576113.048074:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.048076:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576113.048078:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576113.048080:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576113.048084:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576113.048087:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576113.048091:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576113.048099:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393260032:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576113.048107:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.048110:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576113.048113:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0e00 x1875444393260032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.048121:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576113.048124:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576113.048126:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576113.048130:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576113.048132:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576113.048134:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576113.048136:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576113.048139:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:0.0:1788576113.048144:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576113.048146:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576113.048147:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0e00.
00000100:00000001:0.0:1788576113.048151:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576113.048152:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.048155:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.048163:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576113.048165:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576113.048166:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576113.048168:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576113.048170:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576113.048173:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.048181:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576113.048182:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576113.048185:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576113.048187:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576113.048189:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576113.048193:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576113.048196:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576113.048199:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576113.048201:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576113.048204:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576113.048208:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576113.048209:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576113.048214:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576113.048216:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576113.048220:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576113.048224:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393260160, portal 10
00000100:00000001:0.0:1788576113.048226:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576113.048227:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138827904 : -130483570723712 : ffff89536d1e3480)
00000100:00000040:0.0:1788576113.048235:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.048240:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576113.048243:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576113.048246:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576113.048248:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393260160, offset 0
00000400:00000200:0.0:1788576113.048253:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576113.048259:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576113.048265:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788576113.048268:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1788576113.048269:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1788576113.048272:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1788576113.048275:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1788576113.048277:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576113.048281:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000010:3.0:1788576113.048285:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb000.
00000400:00000200:0.0:1788576113.048286:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576113.048291:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331288:331288:256:4294967295] 192.168.202.110@tcp LPNI seq info [331245:331245:8:4294967295]
00000100:00000001:3.0:1788576113.048293:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576113.048300:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1788576113.048304:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1788576113.048316:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1788576113.048321:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000800:00000200:0.0:1788576113.048326:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1788576113.048327:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1788576113.048332:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000100:00000040:3.0:1788576113.048336:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1788576113.048341:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576113.048347:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:3.0:1788576113.048354:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1788576113.048356:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1788576113.048358:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1788576113.048361:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:3.0:1788576113.048365:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000100:0.0:1788576113.048369:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576113.048372:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000100:00000040:3.0:1788576113.048374:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13054)
00000400:00000200:0.0:1788576113.048378:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576113.048383:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576113.048388:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576113.048390:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576113.049551:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576113.049558:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049568:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576113.049572:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049579:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576113.049580:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576113.049587:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576113.049591:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576113.049596:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576113.049599:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307716650
00000100:00000001:0.0:1788576113.049603:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307717562
00000100:00000001:0.0:1788576113.049605:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307716650
00000100:00000001:0.0:1788576113.049607:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307717562
00000100:00000001:0.0:1788576113.049609:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576113.049611:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576113.049613:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576113.049617:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576113/real 1788576113]  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576113.049627:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576113.049629:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576113.049631:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576113.049634:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576113.049636:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576113.049638:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576113.049641:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576113.049648:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049654:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576113.049656:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.049658:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576113.049660:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576113.049663:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049668:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576113.049673:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049680:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576113.049682:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.049684:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576113.049685:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.049686:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576113.049689:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576113.049692:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576113.049696:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576113.049698:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576113.049702:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576113.049710:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576113.049714:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.049716:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576113.049718:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e3480 x1875444393260160/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049724:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576113.049726:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576113.049729:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576113.049732:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576113.049734:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576113.049735:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576113.049737:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576113.049739:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa000.
02000000:00000001:0.0:1788576113.049742:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576113.049743:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576113.049744:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e3480.
00000100:00000001:0.0:1788576113.049747:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576113.049749:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.049751:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.049756:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576113.049758:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576113.049760:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576113.049762:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576113.049763:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576113.049767:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049774:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576113.049776:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576113.049779:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576113.049782:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576113.049784:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576113.049787:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260288:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576113.049790:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576113.049793:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576113.049795:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576113.049797:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576113.049802:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576113.049804:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576113.049808:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576113.049811:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576113.049815:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576113.049820:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393260288, portal 4
00000100:00000001:0.0:1788576113.049822:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576113.049823:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:0.0:1788576113.049829:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049835:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576113.049837:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576113.049840:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576113.049843:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393260288, offset 0
00000400:00000200:0.0:1788576113.049847:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576113.049851:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576113.049857:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576113.049861:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576113.049864:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576113.049867:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576113.049869:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576113.049872:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331289:331289:256:4294967295] 192.168.202.110@tcp LPNI seq info [331246:331246:8:4294967295]
00000400:00000200:0.0:1788576113.049877:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576113.049896:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576113.049901:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576113.049906:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788576113.049912:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576113.049915:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1788576113.049920:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576113.049923:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1788576113.049927:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576113.049929:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788576113.049933:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576113.049936:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576113.049940:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576113.049942:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576113.049944:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576113.049947:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049954:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576113.049957:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.049963:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576113.049964:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576113.049967:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576113.049969:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576113.049972:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576113.049974:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307716796
00000100:00000001:0.0:1788576113.049977:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307717562
00000100:00000001:0.0:1788576113.049979:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307716796
00000100:00000001:0.0:1788576113.049980:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307717562
00000100:00000001:0.0:1788576113.049982:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576113.049984:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576113.049986:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576113.049990:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576113/real 1788576113]  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576113.049996:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576113.049997:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576113.049999:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576113.050001:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576113.050002:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576113.050004:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576113.050006:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576113.050014:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.050020:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576113.050022:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.050024:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576113.050025:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576113.050028:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.050035:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576113.050039:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.050044:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576113.050046:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.050048:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576113.050049:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.050050:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576113.050052:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576113.050054:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576113.050057:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576113.050059:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576113.050062:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576113.050068:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260288:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576113.050071:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.050073:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576113.050076:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0000 x1875444393260288/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576138 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576113.050082:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576113.050083:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576113.050085:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576113.050087:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576113.050089:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576113.050090:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576113.050091:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576113.050092:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb000.
02000000:00000001:0.0:1788576113.050095:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576113.050096:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576113.050097:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0000.
00000100:00000001:0.0:1788576113.050099:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576113.050100:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.050102:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.050105:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576113.050106:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576113.050108:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576113.050109:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576113.429510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576113.429517:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576113.429523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576113.429524:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576113.429527:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576113.429529:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576113.429660:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576113.429664:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576113.429667:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576113.557511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576113.557520:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576113.557522:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576113.557528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576113.557532:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576113.557536:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.557539:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576113.557541:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576113.557544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576114.005600:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307717562, at 4307717568
00000100:00000010:1.0:1788576114.005615:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d76c0.
00000100:00000040:0.0:1788576114.005980:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307717562, at 4307717568
00000100:00000010:0.0:1788576114.005988:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00010000:00000001:3.0:1788576114.453584:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576114.453591:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576114.453597:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576114.453599:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576114.453603:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576114.453604:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576114.453935:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576114.453939:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576114.453943:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576114.582107:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576114.582113:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576114.582115:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576114.582119:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576114.582121:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576114.582124:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576114.582126:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576114.582128:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576114.582130:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576115.477553:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576115.477612:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576115.477619:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576115.477621:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576115.477625:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576115.477626:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576115.477726:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576115.477729:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576115.477732:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576115.605502:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576115.605512:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576115.605514:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576115.605521:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576115.605525:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576115.605529:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576115.605531:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576115.605534:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576115.605537:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576116.501494:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576116.501500:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576116.501506:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576116.501507:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576116.501511:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576116.501512:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576116.501606:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576116.501609:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576116.501612:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576116.629765:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576116.629773:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576116.629775:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576116.629780:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576116.629783:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576116.629787:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576116.629790:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576116.629792:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576116.629794:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576117.525575:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576117.525584:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576117.525592:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576117.525596:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576117.525599:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576117.525604:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576117.525607:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576117.525610:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576117.525612:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576117.525615:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576117.525619:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576117.525621:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576117.525622:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576117.525623:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576117.525625:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576117.525626:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576117.525628:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576117.525630:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576117.525701:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576117.525705:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576117.525708:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576117.653640:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576117.653648:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576117.653652:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576117.653660:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576117.653665:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576117.653670:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576117.653673:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576117.653676:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576117.653680:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576118.165802:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576118.165810:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788576118.165812:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576118.165816:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576118.165819:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576118.165824:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13049
00000400:00000200:1.0:1788576118.165832:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576118.165838:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576118.165841:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576118.165843:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.165846:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576118.165849:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.165851:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576118.165854:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576118.165857:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788576118.165859:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576118.165863:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788576118.165865:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788576118.165868:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788576118.165871:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.165873:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576118.165875:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.165877:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576118.165880:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788576118.165883:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576118.165886:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576118.165890:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576118.165893:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576118.165907:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576118.165910:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788576118.165912:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576118.165923:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6d80.
00000020:00000040:1.0:1788576118.166479:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788576118.166485:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576118.166487:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576118.166490:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576118.166492:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576118.166496:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576118.166504:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788576118.166511:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576118.166521:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576118.166531:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576118.166535:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576118.166540:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2fa5830] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.166609:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576118.166613:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576118.166616:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576118.166622:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.166630:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576118.166632:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576118.166635:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576118.166637:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576118.166640:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576118.166643:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393260416:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576118.166647:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576118.166651:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576118.166654:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576118.166656:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576118.166661:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576118.166664:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576118.166669:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576118.166682:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576118.166689:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576118.166696:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393260416, portal 25
00000100:00000001:0.0:1788576118.166698:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576118.166700:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345477504 : -130483364074112 : ffff8953796f6d80)
00000100:00000040:0.0:1788576118.166706:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.166714:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576118.166717:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576118.166720:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576118.166723:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393260416, offset 0
00000400:00000200:0.0:1788576118.166729:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576118.166736:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576118.166742:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576118.166748:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576118.166752:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576118.166755:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576118.166759:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576118.166762:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331290:331290:256:4294967295] 192.168.202.110@tcp LPNI seq info [331247:331247:8:4294967295]
00000400:00000200:0.0:1788576118.166767:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576118.166776:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576118.166781:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576118.166787:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788576118.166793:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576118.166797:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788576118.166802:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576118.166805:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788576118.166809:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576118.166812:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788576118.166818:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576118.166822:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576118.166828:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576118.166830:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576118.166834:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576118.166838:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576118.166942:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576118.166946:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576118.166950:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788576118.166951:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576118.166953:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576118.166956:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576118.166959:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13049
00000400:00000200:1.0:1788576118.166963:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576118.166967:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576118.166969:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576118.166972:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.166974:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576118.166976:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.166978:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576118.166980:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576118.166982:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788576118.166984:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576118.166986:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788576118.166988:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788576118.166990:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788576118.166994:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.166995:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576118.166997:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.166999:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576118.167001:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788576118.167004:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576118.167006:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576118.167009:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576118.167014:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576118.167016:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576118.167018:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576118.167022:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788576118.167025:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576118.167026:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576118.167030:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f6300.
00000020:00000040:1.0:1788576118.167033:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788576118.167035:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576118.167036:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576118.167038:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576118.167039:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576118.167041:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576118.167045:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975b400.
00000100:00000001:1.0:1788576118.167048:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576118.167053:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576118.167061:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576118.167063:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576118.167066:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5d0841d] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167170:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576118.167184:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576143 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167192:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.167194:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576118.167201:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576118.167205:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576118.167207:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307722682
00000100:00000010:0.0:1788576118.167212:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7300.
00000100:00000001:0.0:1788576118.167218:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1788576118.167225:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307722682
00000100:00000010:0.0:1788576118.167228:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8953656d7de0.
00000100:00000001:0.0:1788576118.167231:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1788576118.167234:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576118.167236:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.167239:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576118.167243:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576118/real 1788576118]  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576118.167260:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576118.167261:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576118.167264:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576118.167266:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576118.167268:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576118.167271:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576118.167274:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576118.167282:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167288:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576118.167290:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167292:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576118.167293:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576118.167296:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167300:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576118.167303:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167307:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576118.167309:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167310:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576118.167311:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167312:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576118.167313:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576118.167315:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576118.167317:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576118.167319:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576118.167321:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576118.167326:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393260416:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576118.167329:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167331:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576118.167333:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6d80 x1875444393260416/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167339:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576118.167342:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576118.167344:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576118.167347:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576118.167349:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576118.167350:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1788576118.167352:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576118.167354:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:0.0:1788576118.167357:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576118.167359:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576118.167361:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6d80.
00000100:00000001:0.0:1788576118.167364:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576118.167364:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167366:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167372:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576118.167374:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.167376:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576118.167378:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576118.167379:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576118.167383:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167390:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576118.167434:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576118.167437:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576118.167439:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576118.167441:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576118.167444:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260544:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576118.167447:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576118.167449:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576118.167451:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576118.167452:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576118.167456:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576118.167457:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576118.167461:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576118.167464:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576118.167467:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576118.167470:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393260544, portal 10
00000100:00000001:0.0:1788576118.167472:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576118.167473:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345474816 : -130483364076800 : ffff8953796f6300)
00000100:00000040:0.0:1788576118.167480:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167486:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576118.167488:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576118.167491:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576118.167493:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393260544, offset 0
00000400:00000200:0.0:1788576118.167496:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576118.167500:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576118.167505:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576118.167508:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576118.167510:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576118.167513:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576118.167515:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576118.167518:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331291:331291:256:4294967295] 192.168.202.110@tcp LPNI seq info [331248:331248:8:4294967295]
00000400:00000200:0.0:1788576118.167522:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576118.167528:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576118.167534:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576118.167537:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788576118.167541:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576118.167544:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788576118.167548:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576118.167551:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788576118.167554:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576118.167556:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788576118.167559:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576118.167563:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576118.167566:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576118.167568:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576118.167570:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576118.167573:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167581:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576118.167584:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576143 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167591:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.167592:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576118.167595:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576118.167597:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:0.0:1788576118.167599:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576118.167601:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307721767
00000100:00000001:0.0:1788576118.167602:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307722682
00000100:00000001:0.0:1788576118.167604:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307721767
00000100:00000001:0.0:1788576118.167605:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307722682
00000100:00000001:0.0:1788576118.167606:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576118.167608:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.167609:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576118.167611:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576118/real 1788576118]  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576118.167615:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576118.167616:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576118.167618:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576118.167619:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576118.167620:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576118.167622:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576118.167624:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576118.167629:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167632:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576118.167633:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167634:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576118.167635:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576118.167637:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167640:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576118.167642:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167646:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576118.167647:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167648:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576118.167648:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167649:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576118.167651:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576118.167653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576118.167655:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576118.167656:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576118.167658:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576118.167663:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260544:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1788576118.167665:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167667:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576118.167668:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f6300 x1875444393260544/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.167672:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576118.167673:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576118.167675:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576118.167688:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576118.167690:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576118.167691:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:0.0:1788576118.167693:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576118.167694:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975b400.
02000000:00000001:0.0:1788576118.167697:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576118.167698:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576118.167699:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f6300.
00000100:00000001:0.0:1788576118.167701:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576118.167702:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167704:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.167707:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576118.167708:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.167710:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576118.167712:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576118.167949:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576118.167952:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576118.167956:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788576118.167957:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576118.167959:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576118.167961:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576118.167964:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13049
00000400:00000200:1.0:1788576118.167968:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576118.167972:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576118.167974:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576118.167978:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.167980:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576118.167982:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.167984:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576118.167986:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576118.167988:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788576118.167989:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576118.167991:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788576118.167993:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788576118.167996:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788576118.167998:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.168000:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576118.168002:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576118.168004:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576118.168007:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788576118.168009:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576118.168012:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576118.168014:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576118.168017:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576118.168019:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576118.168021:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576118.168024:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788576118.168026:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.168027:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1788576118.168028:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.168029:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576118.168032:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5880.
00000020:00000040:1.0:1788576118.168034:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788576118.168037:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576118.168038:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576118.168040:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576118.168041:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576118.168043:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576118.168048:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975a800.
00000100:00000001:1.0:1788576118.168052:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576118.168057:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576118.168065:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576118.168067:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576118.168070:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001001742b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576118.168087:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576118.168091:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788576118.168094:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788576118.168099:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13059)
00000100:00000001:0.0:1788576118.168114:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576118.168116:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576118.168118:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1788576118.168122:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.168129:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576118.168130:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576118.168132:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1788576118.168133:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576118.168136:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576118.168139:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260672:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576118.168142:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1788576118.168144:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1788576118.168146:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576118.168147:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1788576118.168151:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576118.168153:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576118.168156:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1788576118.168159:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8953488daf78.
00000400:00000010:0.0:1788576118.168162:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d4660.
00000100:00000200:0.0:1788576118.168165:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393260672, portal 4
00000100:00000001:0.0:1788576118.168167:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1788576118.168168:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345472128 : -130483364079488 : ffff8953796f5880)
00000100:00000040:0.0:1788576118.168183:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.168189:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1788576118.168191:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1788576118.168194:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff89536d1d43b8.
00000100:00000200:0.0:1788576118.168197:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393260672, offset 0
00000400:00000200:0.0:1788576118.168201:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1788576118.168205:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576118.168209:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1788576118.168212:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788576118.168214:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1788576118.168217:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788576118.168219:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1788576118.168222:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331292:331292:256:4294967295] 192.168.202.110@tcp LPNI seq info [331249:331249:8:4294967295]
00000400:00000200:0.0:1788576118.168225:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576118.168231:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788576118.168235:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576118.168239:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff895349af9800.
00000800:00000200:0.0:1788576118.168243:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1788576118.168246:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1788576118.168250:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1788576118.168252:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1788576118.168256:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1788576118.168258:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff895349af9800.
00000400:00000200:0.0:1788576118.168261:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1788576118.168265:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1788576118.168268:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d43b8
00000400:00000010:0.0:1788576118.168270:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d43b8.
00000100:00000001:0.0:1788576118.168272:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:0.0:1788576118.168276:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576143 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.168283:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576118.168286:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576143 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.168292:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.168293:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:0.0:1788576118.168296:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576118.168299:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:0.0:1788576118.168302:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576118.168304:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307721914
00000100:00000001:0.0:1788576118.168307:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307722682
00000100:00000001:0.0:1788576118.168309:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307721914
00000100:00000001:0.0:1788576118.168311:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307722682
00000100:00000001:0.0:1788576118.168312:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1788576118.168314:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.168316:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1788576118.168320:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576118/real 1788576118]  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1788576118.168328:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1788576118.168329:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1788576118.168331:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8953488daf78.
00000400:00000200:0.0:1788576118.168333:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff89536d1d4660
00000400:00000010:0.0:1788576118.168334:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff89536d1d4660.
00000100:00000001:0.0:1788576118.168336:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:0.0:1788576118.168340:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788576118.168347:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.168353:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:0.0:1788576118.168355:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.168356:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576118.168358:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1788576118.168361:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.168367:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1788576118.168371:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.168378:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1788576118.168380:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.168382:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1788576118.168383:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.168385:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1788576118.168387:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1788576118.168389:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1788576118.168408:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1788576118.168411:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1788576118.168415:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1788576118.168423:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260672:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1788576118.168427:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.168428:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1788576118.168431:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5880 x1875444393260672/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576143 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576118.168438:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1788576118.168439:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1788576118.168441:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cd4000.
02000000:00000001:0.0:1788576118.168443:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1788576118.168445:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1788576118.168446:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:0.0:1788576118.168448:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1788576118.168450:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975a800.
02000000:00000001:0.0:1788576118.168452:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1788576118.168453:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1788576118.168455:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5880.
00000100:00000001:0.0:1788576118.168457:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1788576118.168459:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.168460:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.168464:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576118.168465:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576118.168467:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1788576118.168468:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576118.549518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576118.549525:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576118.549531:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576118.549532:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576118.549535:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576118.549536:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576118.549771:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576118.549775:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576118.549778:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576118.678167:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576118.678173:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576118.678174:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576118.678178:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576118.678181:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576118.678184:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.678186:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576118.678187:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576118.678189:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576119.125514:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307722682, at 4307722688
00000100:00000010:1.0:1788576119.125531:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7de0.
00000100:00000040:0.0:1788576119.130117:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307722682, at 4307722692
00000100:00000010:0.0:1788576119.130134:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8953656d7300.
00010000:00000001:3.0:1788576119.574225:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576119.574234:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576119.574237:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576119.574240:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576119.574245:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576119.574248:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576119.574250:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576119.574252:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576119.574256:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576119.574258:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576119.574262:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576119.574263:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576119.574324:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576119.574328:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576119.574332:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576119.701511:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576119.701518:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576119.701519:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576119.701524:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576119.701527:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576119.701530:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576119.701532:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576119.701533:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576119.701536:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576120.597520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576120.597528:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576120.597534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576120.597536:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576120.597539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576120.597541:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576120.599101:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576120.599106:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576120.599110:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576120.725514:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576120.725521:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576120.725523:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576120.725528:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576120.725531:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576120.725535:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576120.725538:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576120.725540:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576120.725544:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576121.621528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576121.621534:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576121.621539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576121.621541:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576121.621544:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576121.621545:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576121.624015:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576121.624022:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576121.624027:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576121.749537:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576121.749545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576121.749547:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576121.749552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576121.749556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576121.749559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576121.749562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576121.749564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576121.749567:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576122.645528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576122.645535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576122.645540:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576122.645541:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576122.645544:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576122.645546:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576122.645958:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576122.645962:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576122.645965:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576122.773690:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576122.773697:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576122.773699:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576122.773704:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576122.773708:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576122.773723:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576122.773726:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576122.773729:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576122.773732:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576123.285750:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576123.285764:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788576123.285767:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576123.285772:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576123.285779:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576123.285787:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13054
00000400:00000200:0.0:1788576123.285798:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576123.285808:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576123.285811:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576123.285815:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.285819:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576123.285822:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.285825:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576123.285830:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576123.285832:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788576123.285835:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576123.285839:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788576123.285841:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788576123.285844:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788576123.285847:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.285849:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576123.285851:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.285853:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576123.285857:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788576123.285861:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576123.285863:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576123.285867:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576123.285870:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576123.285873:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576123.285877:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576123.285880:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576123.285892:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710700.
00000020:00000040:0.0:1788576123.285897:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788576123.285901:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576123.285903:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576123.285906:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576123.285923:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576123.285928:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576123.285938:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788576123.285947:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576123.285959:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576123.285971:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576123.285976:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576123.285981:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000812ac197] to pc [ptlrpcd_rcv+-1]  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286051:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576123.286065:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576123.286068:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576123.286076:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286084:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576123.286087:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576123.286091:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576123.286094:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576123.286097:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576123.286100:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393260800:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576123.286105:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576123.286110:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576123.286113:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576123.286115:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576123.286120:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576123.286122:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576123.286129:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576123.286134:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576123.286142:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576123.286151:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393260800, portal 25
00000100:00000001:2.0:1788576123.286154:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576123.286156:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523498752 : -130484186052864 : ffff895348710700)
00000100:00000040:2.0:1788576123.286162:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286170:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576123.286174:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576123.286176:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576123.286180:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393260800, offset 0
00000400:00000200:2.0:1788576123.286185:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576123.286194:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576123.286200:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576123.286205:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576123.286211:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576123.286215:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576123.286218:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576123.286222:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331293:331293:256:4294967295] 192.168.202.110@tcp LPNI seq info [331250:331250:8:4294967295]
00000400:00000200:2.0:1788576123.286227:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576123.286237:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576123.286244:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576123.286250:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576123.286258:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576123.286262:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788576123.286268:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576123.286272:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1788576123.286276:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576123.286279:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576123.286289:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576123.286295:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576123.286302:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576123.286305:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576123.286310:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576123.286314:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576123.286341:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576123.286345:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576123.286349:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788576123.286350:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576123.286352:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576123.286355:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576123.286358:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13054
00000400:00000200:0.0:1788576123.286363:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576123.286366:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576123.286368:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576123.286370:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.286373:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576123.286375:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.286377:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576123.286381:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576123.286382:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788576123.286383:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576123.286386:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788576123.286388:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788576123.286390:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788576123.286424:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.286426:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576123.286427:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.286429:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576123.286430:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788576123.286432:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576123.286433:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576123.286434:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576123.286440:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576123.286442:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576123.286443:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576123.286448:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788576123.286450:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576123.286451:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576123.286454:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710a80.
00000020:00000040:0.0:1788576123.286456:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788576123.286458:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576123.286459:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576123.286460:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576123.286461:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576123.286463:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576123.286466:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788576123.286468:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576123.286472:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576123.286486:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576123.286492:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576123.286496:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008406583] to pc [ptlrpcd_rcv+-1]  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286543:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576123.286548:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576148 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286557:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576123.286559:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576123.286567:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576123.286573:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576123.286577:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307727802
00000100:00000010:2.0:1788576123.286582:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cb40.
00000100:00000001:2.0:1788576123.286591:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576123.286598:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307727802
00000100:00000010:2.0:1788576123.286602:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cde0.
00000100:00000001:2.0:1788576123.286605:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576123.286608:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576123.286611:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576123.286614:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576123.286619:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576123/real 1788576123]  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576123.286630:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576123.286632:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576123.286634:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576123.286636:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576123.286637:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576123.286639:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576123.286642:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576123.286648:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286653:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576123.286654:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.286656:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576123.286657:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576123.286660:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286664:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576123.286668:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286674:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576123.286675:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.286676:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576123.286677:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.286679:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576123.286681:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576123.286683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576123.286686:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576123.286688:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576123.286691:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576123.286696:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393260800:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576123.286699:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.286701:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576123.286702:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710700 x1875444393260800/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286706:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576123.286708:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576123.286710:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576123.286712:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576123.286714:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576123.286715:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576123.286717:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576123.286720:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:2.0:1788576123.286724:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576123.286726:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576123.286727:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710700.
00000100:00000001:2.0:1788576123.286730:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576123.286731:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.286734:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.286744:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576123.286746:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576123.286748:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576123.286749:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576123.286751:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576123.286755:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286768:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576123.286770:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576123.286772:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576123.286774:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576123.286776:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576123.286779:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576123.286782:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576123.286784:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576123.286786:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576123.286787:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576123.286790:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576123.286792:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576123.286796:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576123.286799:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576123.286803:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576123.286806:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393260928, portal 10
00000100:00000001:2.0:1788576123.286808:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576123.286809:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523499648 : -130484186051968 : ffff895348710a80)
00000100:00000040:2.0:1788576123.286815:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.286821:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576123.286823:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576123.286825:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576123.286827:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393260928, offset 0
00000400:00000200:2.0:1788576123.286831:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576123.286835:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576123.286839:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576123.286842:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576123.286844:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576123.286847:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576123.286849:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576123.286853:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331294:331294:256:4294967295] 192.168.202.110@tcp LPNI seq info [331251:331251:8:4294967295]
00000400:00000200:2.0:1788576123.286856:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576123.286863:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576123.286867:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576123.286870:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576123.286874:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576123.286876:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788576123.286880:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576123.286883:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1788576123.286958:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576123.286962:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576123.286965:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576123.286967:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576123.286969:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576123.286972:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576123.286975:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13054
00000400:00000200:0.0:1788576123.286979:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576123.286982:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576123.286985:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576123.286987:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.286990:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576123.286991:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.286993:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576123.286996:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576123.286997:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788576123.286999:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576123.287001:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788576123.287003:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788576123.287005:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788576123.287008:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.287010:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576123.287012:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576123.287014:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576123.287017:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788576123.287019:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576123.287023:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576123.287024:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576123.287028:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576123.287030:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576123.287032:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576123.287034:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788576123.287037:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576123.287039:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576123.287042:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710000.
00000020:00000040:0.0:1788576123.287045:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788576123.287047:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576123.287049:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576123.287051:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576123.287052:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576123.287054:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576123.287058:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000100:00000001:0.0:1788576123.287062:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576123.287067:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576123.287073:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576123.287075:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576123.287079:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000045c30881] to pc [ptlrpcd_rcv+-1]  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576123.287086:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576123.287089:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788576123.287092:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788576123.287097:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13064)
00000800:00000100:2.0:1788576123.287382:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576123.287388:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576123.287425:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576123.287430:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576123.287439:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576123.287441:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576123.287445:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576123.287453:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287465:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576123.287469:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576148 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287476:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576123.287478:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576123.287486:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576123.287490:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576123.287496:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576123.287499:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307726888
00000100:00000001:2.0:1788576123.287502:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307727802
00000100:00000001:2.0:1788576123.287505:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307726888
00000100:00000001:2.0:1788576123.287507:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307727802
00000100:00000001:2.0:1788576123.287509:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576123.287512:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576123.287516:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576123.287520:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576123/real 1788576123]  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576123.287530:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576123.287532:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576123.287534:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576123.287538:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576123.287539:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576123.287542:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576123.287545:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576123.287552:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287558:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576123.287560:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.287562:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576123.287564:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576123.287567:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287573:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576123.287577:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287584:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576123.287587:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.287589:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576123.287590:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.287592:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576123.287595:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576123.287598:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576123.287603:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576123.287605:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576123.287609:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576123.287616:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393260928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576123.287621:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.287624:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576123.287626:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710a80 x1875444393260928/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287632:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576123.287635:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576123.287639:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576123.287642:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576123.287644:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576123.287645:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576123.287648:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576123.287650:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:2.0:1788576123.287653:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576123.287655:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576123.287656:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710a80.
00000100:00000001:2.0:1788576123.287659:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576123.287660:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.287662:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.287670:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576123.287671:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576123.287674:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576123.287676:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576123.287677:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576123.287681:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287687:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576123.287689:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576123.287692:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576123.287695:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576123.287697:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576123.287700:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576123.287704:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576123.287708:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576123.287710:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576123.287712:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576123.287719:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576123.287721:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576123.287725:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576123.287728:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576123.287733:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576123.287738:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393261056, portal 4
00000100:00000001:2.0:1788576123.287741:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576123.287742:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523496960 : -130484186054656 : ffff895348710000)
00000100:00000040:2.0:1788576123.287749:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287755:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576123.287758:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576123.287760:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576123.287763:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393261056, offset 0
00000400:00000200:2.0:1788576123.287768:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576123.287775:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576123.287782:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576123.287787:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576123.287791:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576123.287794:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576123.287796:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576123.287801:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331295:331295:256:4294967295] 192.168.202.110@tcp LPNI seq info [331252:331252:8:4294967295]
00000400:00000200:2.0:1788576123.287806:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576123.287814:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576123.287819:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576123.287823:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576123.287828:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576123.287831:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1788576123.287836:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576123.287838:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1788576123.287842:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576123.287843:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576123.287846:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576123.287849:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576123.287852:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576123.287854:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576123.287856:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576123.287860:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576148 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287867:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576123.287870:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576148 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.287876:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576123.287877:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576123.287881:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576123.287884:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576123.287887:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576123.287889:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307727034
00000100:00000001:2.0:1788576123.287891:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307727802
00000100:00000001:2.0:1788576123.287893:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307727034
00000100:00000001:2.0:1788576123.287896:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307727802
00000100:00000001:2.0:1788576123.287898:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576123.287900:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576123.287901:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576123.287905:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576123/real 1788576123]  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576123.288298:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576123.288299:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576123.288302:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576123.288305:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576123.288360:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576123.288366:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576123.288371:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576123.288380:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.288386:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576123.288388:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.288390:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576123.288414:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576123.288419:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.288426:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576123.288431:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.288437:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576123.288440:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.288442:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576123.288444:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.288447:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576123.288449:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576123.288452:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576123.288456:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576123.288460:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576123.288465:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576123.288473:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576123.288477:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.288479:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576123.288481:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710000 x1875444393261056/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576148 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576123.288487:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576123.288489:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576123.288490:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10400.
02000000:00000001:2.0:1788576123.288493:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576123.288495:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576123.288497:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576123.288499:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576123.288501:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788576123.288503:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576123.288504:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576123.288506:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710000.
00000100:00000001:2.0:1788576123.288508:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576123.288510:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.288512:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576123.288516:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576123.288518:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576123.288520:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576123.288521:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576123.670155:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576123.670165:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576123.670172:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576123.670173:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576123.670178:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576123.670179:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576123.670973:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576123.670978:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576123.670982:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576123.797534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576123.797546:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576123.797548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576123.797555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576123.797560:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576123.797566:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576123.797569:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576123.797572:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576123.797575:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576124.245525:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307727802, at 4307727808
00000100:00000040:1.0:1788576124.245525:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307727802, at 4307727808
00000100:00000010:0.0:1788576124.245539:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cb40.
00000100:00000010:1.0:1788576124.245553:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cde0.
00010000:00000001:3.0:1788576124.693577:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576124.693586:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576124.693593:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576124.693595:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576124.693598:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576124.693600:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576124.693687:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576124.693692:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576124.693695:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576124.821588:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576124.821597:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576124.821600:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576124.821607:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576124.821612:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576124.821615:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576124.821617:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576124.821619:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576124.821622:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576125.717581:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576125.717592:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576125.717599:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576125.717600:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576125.717605:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576125.717606:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576125.717680:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576125.717685:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576125.717689:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576125.845525:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576125.845532:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576125.845534:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576125.845539:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576125.845542:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576125.845547:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576125.845549:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576125.845551:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576125.845554:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576126.741467:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576126.741477:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576126.741486:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576126.741490:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576126.741493:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576126.741508:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576126.741512:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576126.741515:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576126.741517:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576126.741635:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576126.741646:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576126.741649:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576126.741651:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576126.741654:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576126.741657:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576126.741659:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576126.741661:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576126.741663:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576126.741841:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576126.741847:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576126.741851:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576126.869584:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576126.869592:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576126.869594:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576126.869600:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576126.869604:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576126.869608:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576126.869610:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576126.869613:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576126.869615:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576127.765609:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576127.765617:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576127.765623:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576127.765624:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576127.765627:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576127.765629:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576127.765888:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576127.765893:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576127.765896:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576127.893531:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576127.893540:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576127.893542:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576127.893548:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576127.893551:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576127.893555:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576127.893557:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576127.893560:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576127.893563:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576128.405566:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576128.405582:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788576128.405585:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576128.405589:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576128.405593:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576128.405600:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13059
00000400:00000200:0.0:1788576128.405608:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576128.405614:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576128.405617:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576128.405620:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405623:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576128.405626:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405628:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576128.405631:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576128.405633:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788576128.405636:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576128.405639:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788576128.405641:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788576128.405643:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788576128.405645:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405647:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576128.405650:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405651:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576128.405654:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788576128.405657:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405660:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576128.405672:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576128.405674:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576128.405677:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576128.405679:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576128.405681:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576128.405690:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348711180.
00000020:00000040:0.0:1788576128.405694:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788576128.405697:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576128.405699:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576128.405702:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576128.405704:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576128.405708:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576128.405715:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7c00.
00000100:00000001:0.0:1788576128.405721:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576128.405731:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576128.405739:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576128.405743:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576128.405747:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b703d06] to pc [ptlrpcd_rcv+-1]  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576128.405770:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576128.405773:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576128.405776:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788576128.405778:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576128.405779:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576128.405781:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576128.405784:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13059
00000400:00000200:0.0:1788576128.405788:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576128.405791:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576128.405792:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576128.405795:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405797:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576128.405799:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405801:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576128.405803:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576128.405804:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788576128.405805:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576128.405807:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788576128.405809:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788576128.405811:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788576128.405813:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405815:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576128.405816:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405818:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576128.405820:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788576128.405822:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576128.405824:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576128.405826:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576128.405831:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576128.405832:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576128.405834:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576128.405837:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788576128.405840:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576128.405841:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576128.405844:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712d80.
00000020:00000040:0.0:1788576128.405847:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788576128.405849:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576128.405850:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576128.405852:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576128.405853:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576128.405854:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576128.405858:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6400.
00000100:00000001:0.0:1788576128.405862:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576128.405867:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576128.405869:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576128.405873:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576128.405877:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1788576128.405878:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576128.405881:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576128.405887:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788576128.405888:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd3e8f71] to pc [ptlrpcd_rcv+-1]  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.405912:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576128.405915:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576128.405919:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576128.405938:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576128.405941:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576128.405946:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393261184:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1788576128.405949:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576128.405951:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:0.0:1788576128.405954:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:2.0:1788576128.405957:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:0.0:1788576128.405959:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576128.405963:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:2.0:1788576128.405963:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576128.405965:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:2.0:1788576128.405968:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1788576128.405969:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576128.405974:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13059
02000000:00000010:2.0:1788576128.405975:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
00000400:00000200:0.0:1788576128.405980:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1788576128.405981:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1788576128.405985:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1788576128.405989:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576128.405994:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000100:00080000:0.0:1788576128.405997:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576128.406002:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.406004:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576128.406006:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.406007:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000400:00000010:2.0:1788576128.406008:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000020:00000001:0.0:1788576128.406010:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576128.406013:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788576128.406016:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576128.406018:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000100:00000200:2.0:1788576128.406019:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393261184, portal 25
00000020:00000001:0.0:1788576128.406020:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788576128.406024:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000020:00000001:0.0:1788576128.406025:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000001:2.0:1788576128.406028:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523501440 : -130484186050176 : ffff895348711180)
00000100:00000040:0.0:1788576128.406031:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.406034:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576128.406038:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1788576128.406039:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576128.406045:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576128.406049:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788576128.406053:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1788576128.406055:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:2.0:1788576128.406059:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1788576128.406061:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576128.406064:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:2.0:1788576128.406064:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
02000000:00000001:0.0:1788576128.406069:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:2.0:1788576128.406070:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393261184, offset 0
02000000:00000001:0.0:1788576128.406072:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576128.406076:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576128.406079:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000400:00000200:2.0:1788576128.406079:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00000001:0.0:1788576128.406083:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576128.406086:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576128.406090:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000400:00000200:2.0:1788576128.406091:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1788576128.406093:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788576128.406097:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576128.406101:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576128.406101:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1788576128.406105:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576128.406108:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576128.406110:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576128.406110:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576128.406116:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000010:0.0:1788576128.406118:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5000.
00000400:00000200:2.0:1788576128.406121:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1788576128.406123:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576128.406126:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00100000:0.0:1788576128.406132:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1788576128.406134:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331296:331296:256:4294967295] 192.168.202.110@tcp LPNI seq info [331253:331253:8:4294967295]
00000400:00000200:2.0:1788576128.406146:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1788576128.406149:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576128.406153:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576128.406158:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788576128.406163:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1788576128.406174:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1788576128.406175:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1788576128.406178:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788576128.406182:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000010:2.0:1788576128.406183:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000100:00000040:0.0:1788576128.406187:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13069)
00000800:00000200:2.0:1788576128.406192:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576128.406197:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788576128.406202:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576128.406206:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1788576128.406210:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576128.406214:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576128.406221:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576128.406238:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576128.406243:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576128.406246:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576128.406250:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576128.406254:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406265:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576128.406268:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576153 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406276:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576128.406277:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576128.406283:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576128.406287:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576128.406290:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307732921
00000100:00000010:2.0:1788576128.406294:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c120.
00000100:00000001:2.0:1788576128.406300:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576128.406305:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307732921
00000100:00000010:2.0:1788576128.406308:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c780.
00000100:00000001:2.0:1788576128.406311:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576128.406315:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576128.406317:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576128.406319:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576128.406324:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576128/real 1788576128]  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576128.406340:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576128.406342:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576128.406344:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576128.406347:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576128.406348:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576128.406351:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576128.406354:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576128.406362:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406368:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576128.406370:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.406372:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576128.406374:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576128.406378:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406383:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576128.406386:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406408:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576128.406411:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.406413:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576128.406414:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.406416:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576128.406418:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576128.406421:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576128.406426:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576128.406428:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576128.406433:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576128.406441:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393261184:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576128.406446:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.406448:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576128.406450:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348711180 x1875444393261184/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406456:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576128.406459:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576128.406462:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576128.406465:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576128.406467:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576128.406468:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576128.406470:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576128.406472:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7c00.
02000000:00000001:2.0:1788576128.406476:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576128.406477:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576128.406479:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348711180.
00000100:00000001:2.0:1788576128.406482:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576128.406483:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.406485:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.406492:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576128.406494:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576128.406496:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576128.406498:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576128.406499:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576128.406502:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406509:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576128.406510:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576128.406512:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576128.406513:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576128.406516:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576128.406518:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261312:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576128.406522:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576128.406524:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576128.406525:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576128.406526:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576128.406529:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576128.406531:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576128.406535:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576128.406538:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576128.406542:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576128.406545:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393261312, portal 10
00000100:00000001:2.0:1788576128.406547:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576128.406548:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523508608 : -130484186043008 : ffff895348712d80)
00000100:00000040:2.0:1788576128.406554:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406559:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576128.406561:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576128.406564:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576128.406566:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393261312, offset 0
00000400:00000200:2.0:1788576128.406570:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576128.406574:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576128.406577:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576128.406580:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576128.406583:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576128.406586:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576128.406588:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576128.406591:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331297:331297:256:4294967295] 192.168.202.110@tcp LPNI seq info [331254:331254:8:4294967295]
00000400:00000200:2.0:1788576128.406595:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576128.406610:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576128.406615:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576128.406619:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576128.406623:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576128.406626:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788576128.406630:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576128.406633:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1788576128.406637:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576128.406638:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576128.406641:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576128.406645:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576128.406648:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576128.406650:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576128.406652:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576128.406656:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406663:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576128.406666:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576153 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406672:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576128.406674:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576128.406677:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576128.406680:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576128.406683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576128.406685:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307732007
00000100:00000001:2.0:1788576128.406688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307732921
00000100:00000001:2.0:1788576128.406690:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307732007
00000100:00000001:2.0:1788576128.406692:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307732921
00000100:00000001:2.0:1788576128.406693:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576128.406695:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576128.406697:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576128.406701:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576128/real 1788576128]  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576128.406708:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576128.406709:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576128.406711:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576128.406714:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576128.406715:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576128.406717:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576128.406720:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576128.406728:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406735:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576128.406736:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.406738:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576128.406739:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576128.406742:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406748:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576128.406752:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406759:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576128.406762:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.406763:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576128.406764:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.406766:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576128.406768:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576128.406771:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576128.406774:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576128.406777:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576128.406780:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576128.406788:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261312:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576128.406792:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576128.406795:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406801:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576128.406802:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576128.406804:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576128.406806:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576128.406808:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576128.406810:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261440:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576128.406813:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576128.406815:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576128.406817:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576128.406818:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576128.406822:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788576128.406823:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576128.406827:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576128.406829:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576128.406832:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576128.406835:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393261440, portal 4
00000100:00000001:2.0:1788576128.406837:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576128.406838:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:2.0:1788576128.406843:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406849:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576128.406851:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576128.406854:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576128.406856:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393261440, offset 0
00000400:00000200:2.0:1788576128.406860:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576128.406863:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576128.406866:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576128.406869:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576128.406872:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576128.406874:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576128.406876:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576128.406879:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331298:331298:256:4294967295] 192.168.202.110@tcp LPNI seq info [331255:331255:8:4294967295]
00000400:00000200:2.0:1788576128.406883:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576128.406889:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576128.406893:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576128.406897:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576128.406900:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576128.406903:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1788576128.406906:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576128.406909:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1788576128.406913:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576128.406915:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576128.406917:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576128.406932:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576128.406936:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576128.406938:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576128.406940:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576128.406944:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576153 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406951:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576128.406953:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576153 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.406959:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576128.406961:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576128.406964:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576128.406966:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576128.406970:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576128.406972:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307732153
00000100:00000001:2.0:1788576128.406974:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307732921
00000100:00000001:2.0:1788576128.406976:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307732153
00000100:00000001:2.0:1788576128.406978:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307732921
00000100:00000001:2.0:1788576128.406980:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576128.406982:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576128.406983:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576128.406987:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576128/real 1788576128]  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576128.406994:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576128.406995:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576128.406997:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576128.406999:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576128.407001:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576128.407003:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576128.407006:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576128.407013:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.407020:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576128.407021:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.407023:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576128.407024:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576128.407028:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.407034:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576128.407038:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.407044:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576128.407046:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.407048:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576128.407049:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.407051:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576128.407053:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576128.407055:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576128.407058:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576128.407060:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576128.407064:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576128.407071:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261440:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576128.407075:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.407077:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576128.407080:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712d80 x1875444393261312/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.407088:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576128.407090:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576128.407092:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576128.407094:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576128.407096:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576128.407097:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576128.407099:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576128.407101:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6400.
02000000:00000001:2.0:1788576128.407103:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576128.407104:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576128.407106:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712d80.
00000100:00000001:2.0:1788576128.407108:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576128.407110:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.407111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576128.407115:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393261440/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576153 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576128.407121:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576128.407122:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576128.407123:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788576128.407126:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576128.407127:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576128.407128:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576128.407130:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576128.407131:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5000.
02000000:00000001:2.0:1788576128.407133:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576128.407134:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576128.407135:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:2.0:1788576128.407137:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576128.407138:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.407140:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576128.407143:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576128.407144:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576128.407146:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576128.407147:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576128.789579:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576128.789591:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576128.789594:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576128.789598:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576128.789604:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576128.789607:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576128.789610:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576128.789612:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576128.789616:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576128.789618:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576128.789622:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576128.789623:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576128.789741:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576128.789746:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576128.789750:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576128.918352:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576128.918364:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576128.918366:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576128.918374:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576128.918379:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576128.918383:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576128.918386:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576128.918388:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576128.918419:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576129.365558:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307732921, at 4307732928
00000100:00000040:0.0:1788576129.365561:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307732921, at 4307732928
00000100:00000010:1.0:1788576129.365587:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c780.
00000100:00000010:0.0:1788576129.365595:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c120.
00010000:00000001:3.0:1788576129.814124:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576129.814133:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576129.814140:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576129.814141:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576129.814145:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576129.814147:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576129.814252:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576129.814257:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576129.814261:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576129.944622:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576129.944632:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576129.944634:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576129.944640:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576129.944643:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576129.944647:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576129.944650:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576129.944653:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576129.944656:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576130.837558:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576130.837566:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576130.837573:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576130.837575:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576130.837580:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576130.837582:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576130.837752:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576130.837757:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576130.837761:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576130.966045:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576130.966054:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576130.966057:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576130.966064:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576130.966068:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576130.966073:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576130.966077:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576130.966080:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576130.966083:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576131.861557:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576131.861567:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576131.861574:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576131.861576:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576131.861580:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576131.861582:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576131.861756:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576131.861760:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576131.861762:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576131.989835:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576131.989847:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576131.989851:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576131.989859:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576131.989863:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576131.989869:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576131.989872:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576131.989875:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576131.989879:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576132.885524:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576132.885530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576132.885547:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576132.885548:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576132.885552:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576132.885553:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576132.885649:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576132.885655:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576132.885660:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576133.013521:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576133.013528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576133.013531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576133.013536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576133.013540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576133.013544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576133.013547:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576133.013549:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576133.013552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576133.525519:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576133.525531:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788576133.525533:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576133.525537:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576133.525542:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576133.525547:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13064
00000400:00000200:0.0:1788576133.525555:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576133.525560:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576133.525562:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576133.525565:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525567:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576133.525568:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525570:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576133.525573:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576133.525574:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788576133.525576:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576133.525580:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788576133.525581:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788576133.525583:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788576133.525585:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525586:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576133.525587:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525588:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576133.525590:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788576133.525592:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525593:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576133.525595:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576133.525598:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576133.525600:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576133.525601:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576133.525603:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576133.525611:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788576133.525613:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788576133.525616:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576133.525617:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576133.525619:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576133.525621:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576133.525624:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576133.525630:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5800.
00000100:00000001:0.0:1788576133.525635:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576133.525641:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576133.525648:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576133.525651:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576133.525654:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576133.525684:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576133.525687:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576133.525691:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788576133.525693:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576133.525694:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576133.525697:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576133.525700:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13064
00000400:00000200:0.0:1788576133.525703:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576133.525706:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576133.525708:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576133.525711:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525713:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576133.525714:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525716:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576133.525718:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576133.525719:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788576133.525720:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576133.525722:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788576133.525724:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788576133.525726:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788576133.525728:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525729:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576133.525731:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525733:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576133.525735:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788576133.525737:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525739:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576133.525741:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576133.525746:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576133.525748:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576133.525749:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576133.525753:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788576133.525757:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576133.525759:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576133.525762:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788576133.525765:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788576133.525767:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576133.525768:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576133.525770:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576133.525771:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576133.525773:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576133.525776:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788576133.525780:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576133.525785:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576133.525792:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576133.525794:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576133.525797:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576133.525804:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576133.525807:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576133.525810:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576133.525811:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576133.525812:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576133.525815:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576133.525817:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13064
00000400:00000200:0.0:1788576133.525820:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576133.525823:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576133.525825:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576133.525827:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525829:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576133.525831:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525833:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576133.525835:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576133.525836:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788576133.525838:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576133.525840:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788576133.525841:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788576133.525843:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788576133.525846:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525848:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576133.525850:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525851:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576133.525853:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788576133.525856:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576133.525858:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576133.525860:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576133.525862:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576133.525864:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576133.525866:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576133.525867:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788576133.525870:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576133.525871:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576133.525875:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788576133.525877:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788576133.525879:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576133.525880:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576133.525881:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576133.525883:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576133.525885:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576133.525889:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788576133.525894:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576133.525899:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576133.525911:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576133.525913:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576133.525916:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576133.525923:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576133.525926:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788576133.525930:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788576133.525945:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13074)
00000100:00000001:2.0:1788576133.526066:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576133.526150:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576133.526154:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576133.526168:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.526181:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576133.526184:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576133.526188:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576133.526192:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576133.526196:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576133.526200:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393261568:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576133.526204:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576133.526210:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576133.526213:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576133.526216:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576133.526226:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576133.526228:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576133.526237:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576133.526243:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576133.526252:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576133.526262:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393261568, portal 25
00000100:00000001:2.0:1788576133.526265:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576133.526267:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:2.0:1788576133.526273:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576158 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.526281:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576133.526285:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576133.526288:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576133.526292:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393261568, offset 0
00000400:00000200:2.0:1788576133.526300:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576133.526311:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576133.526321:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576133.526328:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576133.526447:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576133.526453:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576133.526456:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576133.526461:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331299:331299:256:4294967295] 192.168.202.110@tcp LPNI seq info [331256:331256:8:4294967295]
00000400:00000200:2.0:1788576133.526467:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576133.526563:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576133.526571:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576133.526578:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576133.526586:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576133.526591:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788576133.526598:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576133.526602:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1788576133.526606:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576133.526609:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576133.526634:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576133.526639:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576133.526647:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576133.526650:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576133.527119:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576133.527127:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576158 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.527147:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576133.527151:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576158 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.527160:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576133.527161:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576133.527171:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576133.527177:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576133.527180:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307738042
00000100:00000010:2.0:1788576133.527188:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c060.
00000100:00000001:2.0:1788576133.527197:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576133.527205:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307738042
00000100:00000010:2.0:1788576133.527208:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c5a0.
00000100:00000001:2.0:1788576133.527213:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576133.527216:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576133.527220:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576133.527223:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576133.527228:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576133/real 1788576133]  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576133.527240:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576133.527242:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576133.527245:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576133.527249:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576133.527250:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576133.527254:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576133.527257:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576133.527265:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.527272:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576133.527275:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.527278:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576133.527280:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576133.527285:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.527292:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576133.527298:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.527305:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576133.527308:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.527310:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576133.527311:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.527314:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576133.527317:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576133.527320:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576133.527327:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576133.527330:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576133.527347:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576133.527356:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393261568:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576133.527362:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576133.527366:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.527373:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576133.527374:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576133.527377:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576133.527380:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576133.527383:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576133.527386:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261696:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576133.527423:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576133.527427:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576133.527430:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576133.527432:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576133.527437:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788576133.527440:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576133.527445:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576133.527448:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576133.527453:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576133.527458:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393261696, portal 10
00000100:00000001:2.0:1788576133.527461:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576133.527462:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00000040:2.0:1788576133.527481:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576158 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.527487:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576133.527490:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576133.527493:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576133.527496:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393261696, offset 0
00000400:00000200:2.0:1788576133.527502:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576133.527509:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576133.527515:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576133.527520:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576133.527523:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576133.527526:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576133.527529:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576133.527532:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331300:331300:256:4294967295] 192.168.202.110@tcp LPNI seq info [331257:331257:8:4294967295]
00000400:00000200:2.0:1788576133.527537:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576133.527545:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576133.527550:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576133.527555:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576133.527559:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576133.527563:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788576133.527568:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576133.527570:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1788576133.527574:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576133.527576:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576133.527580:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576133.527584:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576133.527589:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576133.527591:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576133.527593:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576133.527596:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576158 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.527604:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576133.527606:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576158 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.528798:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576133.528803:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576133.528817:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576133.528824:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576133.528831:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576133.528834:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307737129
00000100:00000001:2.0:1788576133.528839:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307738042
00000100:00000001:2.0:1788576133.528842:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307737129
00000100:00000001:2.0:1788576133.528845:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307738042
00000100:00000001:2.0:1788576133.528847:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576133.528850:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576133.528853:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576133.528863:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576133/real 1788576133]  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576133.528874:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576133.528877:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576133.528881:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576133.528885:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576133.528887:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576133.528891:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576133.528894:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576133.528903:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.528910:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576133.528911:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.528914:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576133.528917:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576133.528921:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.528928:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576133.528933:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.528951:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576133.528955:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.528957:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576133.528958:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.528961:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576133.528964:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576133.528968:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576133.528974:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576133.528977:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576133.528980:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576133.528988:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261696:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576133.528993:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576133.528997:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529004:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576133.529006:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576133.529010:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576133.529013:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576133.529016:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576133.529020:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576133.529024:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576133.529028:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576133.529031:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576133.529033:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576133.529043:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788576133.529045:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576133.529050:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576133.529054:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576133.529058:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576133.529064:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393261824, portal 4
00000100:00000001:2.0:1788576133.529066:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576133.529068:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00000040:2.0:1788576133.529075:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576158 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529082:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576133.529084:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576133.529087:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576133.529090:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393261824, offset 0
00000400:00000200:2.0:1788576133.529096:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576133.529104:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576133.529112:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576133.529117:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576133.529122:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576133.529125:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576133.529129:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576133.529132:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331301:331301:256:4294967295] 192.168.202.110@tcp LPNI seq info [331258:331258:8:4294967295]
00000400:00000200:2.0:1788576133.529137:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576133.529146:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576133.529152:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576133.529157:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576133.529164:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576133.529168:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1788576133.529173:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576133.529177:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1788576133.529181:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576133.529183:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576133.529188:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576133.529192:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576133.529197:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576133.529199:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576133.529202:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576133.529205:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576158 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529213:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576133.529216:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576158 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529222:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576133.529223:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576133.529226:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576133.529229:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576133.529233:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576133.529235:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307737275
00000100:00000001:2.0:1788576133.529237:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307738042
00000100:00000001:2.0:1788576133.529239:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307737275
00000100:00000001:2.0:1788576133.529242:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307738042
00000100:00000001:2.0:1788576133.529244:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576133.529246:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576133.529247:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576133.529251:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576133/real 1788576133]  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576133.529258:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576133.529260:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576133.529261:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576133.529264:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576133.529265:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576133.529268:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576133.529271:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576133.529279:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529285:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576133.529286:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.529288:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576133.529289:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576133.529293:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529300:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576133.529303:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529310:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576133.529313:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.529314:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576133.529315:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.529318:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576133.529320:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576133.529323:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576133.529327:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576133.529329:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576133.529333:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576133.529341:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393261824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576133.529345:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.529348:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576133.529352:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393261568/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529359:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576133.529363:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576133.529367:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576133.529370:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576133.529373:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576133.529375:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576133.529378:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576133.529381:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5800.
02000000:00000001:2.0:1788576133.529387:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576133.529389:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576133.529418:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:2.0:1788576133.529424:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576133.529426:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.529428:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576133.529432:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393261696/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529439:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576133.529440:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576133.529442:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788576133.529445:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576133.529447:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576133.529448:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576133.529450:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576133.529452:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:2.0:1788576133.529454:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576133.529455:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576133.529457:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:2.0:1788576133.529459:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576133.529460:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.529462:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576133.529465:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393261824/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576158 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576133.529472:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576133.529473:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576133.529475:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788576133.529477:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576133.529479:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576133.529480:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576133.529482:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576133.529483:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:2.0:1788576133.529485:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576133.529486:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576133.529488:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:2.0:1788576133.529490:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576133.529491:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.529495:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576133.529505:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576133.529508:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576133.529511:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576133.529513:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576133.909515:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576133.909523:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576133.909530:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576133.909531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576133.909534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576133.909535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576133.909616:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576133.909620:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576133.909623:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576134.037507:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576134.037525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576134.037527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576134.037533:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576134.037537:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576134.037541:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576134.037544:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576134.037546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576134.037549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576134.485533:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307738042, at 4307738048
00000100:00000010:0.0:1788576134.485549:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c060.
00000100:00000040:1.0:1788576134.486056:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307738042, at 4307738048
00000100:00000010:1.0:1788576134.486070:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c5a0.
00000800:00000200:0.0:1788576134.549541:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788576134.549551:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788576134.549556:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1788576134.550265:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1788576134.550281:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1788576134.550296:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1788576134.550304:0:2355:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1788576134.550308:0:2355:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788576134.934267:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576134.934274:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576134.934280:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576134.934281:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576134.934285:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576134.934286:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576134.934384:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576134.934388:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576134.934415:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576135.061520:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576135.061528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576135.061531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576135.061537:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576135.061540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576135.061543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576135.061546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576135.061548:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576135.061551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576135.957540:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576135.957548:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576135.957559:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576135.957560:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576135.957563:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576135.957565:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576135.957677:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576135.957680:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576135.957684:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576136.085483:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576136.085491:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576136.085493:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576136.085499:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576136.085502:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576136.085507:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576136.085510:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576136.085512:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576136.085515:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576136.981509:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576136.981825:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576136.981833:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576136.981836:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576136.981838:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576136.981842:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576136.981845:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576136.981848:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576136.981850:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576136.981852:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576136.981856:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576136.981857:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576136.981858:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576136.981860:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576136.981861:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576136.981862:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576136.981864:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576136.981865:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576136.981973:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576136.981976:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576136.981979:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576137.109534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576137.109541:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576137.109543:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576137.109555:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576137.109558:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576137.109562:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576137.109565:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576137.109567:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576137.109570:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576138.005653:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576138.005660:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576138.005666:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576138.005668:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576138.005671:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576138.005673:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576138.005737:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576138.005741:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576138.005744:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576138.133501:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576138.133509:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576138.133511:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576138.133516:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576138.133519:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576138.133523:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576138.133525:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576138.133528:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576138.133530:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576138.645542:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576138.645551:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788576138.645554:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576138.645558:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576138.645562:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576138.645569:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13069
00000400:00000200:1.0:1788576138.645580:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576138.645586:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576138.645589:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576138.645592:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645595:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576138.645598:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645601:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576138.645605:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576138.645606:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788576138.645609:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576138.645612:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788576138.645614:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788576138.645617:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788576138.645619:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645621:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576138.645623:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645625:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576138.645629:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788576138.645632:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645634:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576138.645637:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576138.645640:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576138.645642:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576138.645644:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788576138.645646:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576138.645656:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f5500.
00000020:00000040:1.0:1788576138.645659:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788576138.645662:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576138.645664:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576138.645667:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576138.645669:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576138.645673:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576138.645680:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759400.
00000100:00000001:1.0:1788576138.645686:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576138.645694:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576138.645704:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576138.645708:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576138.645713:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005968946b] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576138.645739:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576138.645742:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576138.645745:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788576138.645747:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576138.645748:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576138.645751:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576138.645754:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13069
00000400:00000200:1.0:1788576138.645757:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576138.645760:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576138.645762:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576138.645765:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645767:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576138.645768:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645770:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576138.645772:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576138.645774:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788576138.645775:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576138.645777:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788576138.645779:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788576138.645781:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788576138.645784:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645787:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576138.645788:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645791:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576138.645793:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788576138.645795:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645798:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576138.645799:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576138.645804:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576138.645805:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576138.645807:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576138.645811:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788576138.645814:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576138.645815:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576138.645818:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788576138.645820:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788576138.645823:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576138.645824:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576138.645826:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576138.645827:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576138.645829:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576138.645832:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff89537975ac00.
00000100:00000001:1.0:1788576138.645835:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576138.645840:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576138.645847:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576138.645849:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576138.645852:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576138.645860:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576138.645862:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576138.645865:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788576138.645867:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576138.645868:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576138.645871:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576138.645873:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13069
00000400:00000200:1.0:1788576138.645877:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576138.645880:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576138.645882:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576138.645885:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645887:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576138.645889:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645891:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576138.645892:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576138.645894:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788576138.645895:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576138.645897:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788576138.645898:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788576138.645900:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788576138.645902:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645904:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576138.645906:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645907:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576138.645910:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788576138.645912:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576138.645915:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576138.645917:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576138.645919:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576138.645921:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576138.645922:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576138.645924:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788576138.645926:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576138.645928:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576138.645930:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788576138.645932:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788576138.645935:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576138.645936:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576138.645937:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576138.645938:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576138.645940:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576138.645942:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758400.
00000100:00000001:1.0:1788576138.645945:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576138.645961:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576138.645969:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576138.645971:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576138.645974:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576138.645982:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576138.645984:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788576138.645988:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788576138.645993:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13079)
00000100:00000001:2.0:1788576138.646752:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576138.646757:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576138.646760:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576138.646767:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.646776:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576138.646779:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576138.646782:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576138.646784:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576138.646786:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576138.646790:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393261952:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576138.646793:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576138.646797:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576138.646800:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576138.646803:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576138.646808:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788576138.646810:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576138.646818:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576138.646821:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576138.646828:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576138.646833:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393261952, portal 25
00000100:00000001:2.0:1788576138.646836:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576138.646837:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345471232 : -130483364080384 : ffff8953796f5500)
00000100:00000040:2.0:1788576138.646843:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576163 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.646850:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576138.646853:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576138.646855:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576138.646859:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393261952, offset 0
00000400:00000200:2.0:1788576138.646866:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576138.646872:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576138.646879:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576138.646884:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576138.646888:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576138.646891:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576138.646893:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576138.646897:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331302:331302:256:4294967295] 192.168.202.110@tcp LPNI seq info [331259:331259:8:4294967295]
00000400:00000200:2.0:1788576138.646902:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576138.646911:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576138.646918:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576138.646924:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576138.646930:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576138.646933:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788576138.646939:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576138.646942:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788576138.646946:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576138.646961:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576138.646968:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576138.646973:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576138.646979:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576138.646982:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576138.646986:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576138.646990:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576163 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647000:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576138.647003:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576163 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647010:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576138.647012:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576138.647018:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576138.647022:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576138.647025:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307743162
00000100:00000010:2.0:1788576138.647030:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cea0.
00000100:00000001:2.0:1788576138.647036:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576138.647042:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307743162
00000100:00000010:2.0:1788576138.647045:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c180.
00000100:00000001:2.0:1788576138.647049:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576138.647052:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576138.647054:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576138.647057:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576138.647061:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576138/real 1788576138]  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576138.647077:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576138.647079:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576138.647081:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576138.647084:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576138.647085:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576138.647088:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576138.647092:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576138.647100:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647107:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576138.647108:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647111:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576138.647113:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576138.647117:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647124:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576138.647128:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647134:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576138.647137:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647138:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576138.647140:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647141:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576138.647143:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576138.647146:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576138.647150:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576138.647152:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576138.647156:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576138.647164:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393261952:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576138.647168:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576138.647171:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647176:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576138.647178:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576138.647180:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576138.647182:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576138.647184:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576138.647186:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576138.647189:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576138.647191:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576138.647192:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576138.647194:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576138.647197:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576138.647199:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576138.647202:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576138.647205:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576138.647208:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576138.647211:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393262080, portal 10
00000100:00000001:2.0:1788576138.647213:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576138.647214:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345469440 : -130483364082176 : ffff8953796f4e00)
00000100:00000040:2.0:1788576138.647220:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576163 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647226:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576138.647228:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576138.647230:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576138.647232:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393262080, offset 0
00000400:00000200:2.0:1788576138.647236:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576138.647240:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576138.647244:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576138.647247:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576138.647249:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576138.647252:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576138.647254:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576138.647257:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331303:331303:256:4294967295] 192.168.202.110@tcp LPNI seq info [331260:331260:8:4294967295]
00000400:00000200:2.0:1788576138.647261:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576138.647267:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576138.647271:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576138.647275:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576138.647279:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576138.647282:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788576138.647286:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576138.647288:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788576138.647292:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576138.647294:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576138.647296:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576138.647300:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576138.647303:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576138.647305:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576138.647307:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576138.647311:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576163 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647319:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576138.647322:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576163 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647328:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576138.647329:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576138.647332:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576138.647336:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576138.647339:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576138.647342:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307742247
00000100:00000001:2.0:1788576138.647344:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307743162
00000100:00000001:2.0:1788576138.647346:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307742247
00000100:00000001:2.0:1788576138.647347:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307743162
00000100:00000001:2.0:1788576138.647349:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576138.647351:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576138.647353:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576138.647357:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576138/real 1788576138]  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576138.647364:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576138.647366:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576138.647367:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576138.647370:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576138.647371:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576138.647373:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576138.647377:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576138.647384:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647389:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576138.647423:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647426:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576138.647428:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576138.647432:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647439:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576138.647454:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647463:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576138.647466:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647468:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576138.647469:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647471:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576138.647473:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576138.647475:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576138.647478:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576138.647481:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576138.647485:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576138.647493:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576138.647496:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576138.647500:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647506:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576138.647508:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576138.647510:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576138.647511:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576138.647513:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576138.647516:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262208:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576138.647518:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576138.647520:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576138.647522:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576138.647523:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576138.647527:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576138.647529:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576138.647532:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576138.647535:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576138.647538:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576138.647540:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393262208, portal 4
00000100:00000001:2.0:1788576138.647543:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576138.647544:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345481088 : -130483364070528 : ffff8953796f7b80)
00000100:00000040:2.0:1788576138.647549:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576163 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647554:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576138.647556:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576138.647558:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576138.647560:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393262208, offset 0
00000400:00000200:2.0:1788576138.647564:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576138.647567:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576138.647571:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576138.647573:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576138.647576:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576138.647578:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576138.647580:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576138.647583:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331304:331304:256:4294967295] 192.168.202.110@tcp LPNI seq info [331261:331261:8:4294967295]
00000400:00000200:2.0:1788576138.647587:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576138.647593:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576138.647607:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576138.647611:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576138.647615:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576138.647617:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1788576138.647621:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576138.647624:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1788576138.647627:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576138.647629:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576138.647631:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576138.647635:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576138.647638:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576138.647640:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576138.647642:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576138.647645:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576163 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647657:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576138.647660:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576163 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647666:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576138.647667:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576138.647670:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576138.647672:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576138.647676:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576138.647677:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307742394
00000100:00000001:2.0:1788576138.647680:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307743162
00000100:00000001:2.0:1788576138.647681:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307742394
00000100:00000001:2.0:1788576138.647683:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307743162
00000100:00000001:2.0:1788576138.647685:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576138.647687:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576138.647688:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576138.647692:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576138/real 1788576138]  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576138.647699:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576138.647700:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576138.647701:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576138.647703:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576138.647704:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576138.647706:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576138.647708:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576138.647715:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647721:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576138.647723:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647725:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576138.647726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576138.647729:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647735:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576138.647738:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647745:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576138.647747:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647748:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576138.647749:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647751:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576138.647752:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576138.647755:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576138.647758:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576138.647761:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576138.647764:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576138.647771:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262208:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576138.647774:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647776:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576138.647780:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f5500 x1875444393261952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647786:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576138.647788:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576138.647791:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788576138.647794:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576138.647796:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576138.647798:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576138.647800:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576138.647803:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759400.
02000000:00000001:2.0:1788576138.647806:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576138.647808:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576138.647809:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f5500.
00000100:00000001:2.0:1788576138.647812:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576138.647814:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647815:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576138.647818:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4e00 x1875444393262080/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647824:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576138.647825:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576138.647827:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576138.647828:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576138.647829:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576138.647830:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576138.647832:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576138.647833:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff89537975ac00.
02000000:00000001:2.0:1788576138.647835:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576138.647836:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576138.647837:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4e00.
00000100:00000001:2.0:1788576138.647839:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576138.647840:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647842:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576138.647845:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7b80 x1875444393262208/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576163 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576138.647851:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576138.647852:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576138.647853:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576138.647855:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576138.647856:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576138.647858:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576138.647859:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576138.647861:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758400.
02000000:00000001:2.0:1788576138.647863:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576138.647865:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576138.647866:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7b80.
00000100:00000001:2.0:1788576138.647868:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576138.647869:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647872:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576138.647878:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576138.647880:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576138.647882:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576138.647884:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576139.030405:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576139.030414:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576139.030416:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576139.030419:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576139.030424:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576139.030426:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576139.030428:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576139.030430:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576139.030434:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576139.030435:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576139.030438:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576139.030440:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576139.030520:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576139.030524:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576139.030528:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576139.159671:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576139.159677:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576139.159678:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576139.159683:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576139.159687:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576139.159691:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576139.159693:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576139.159696:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576139.159699:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576139.605504:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307743162, at 4307743168
00000100:00000010:1.0:1788576139.605519:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c180.
00000100:00000040:0.0:1788576139.605755:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307743162, at 4307743168
00000100:00000010:0.0:1788576139.605760:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cea0.
00010000:00000001:3.0:1788576140.055322:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576140.055328:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576140.055332:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576140.055333:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576140.055335:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576140.055336:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576140.055384:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576140.055388:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576140.055390:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576140.182029:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576140.182036:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576140.182038:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576140.182044:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576140.182047:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576140.182051:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576140.182053:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576140.182056:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576140.182059:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576141.077515:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576141.077520:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576141.077525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576141.077527:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576141.077529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576141.077531:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576141.077589:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576141.077594:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576141.077598:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576141.205522:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576141.205529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576141.205531:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576141.205535:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576141.205539:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576141.205543:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576141.205545:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576141.205547:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576141.205551:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576142.101536:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576142.101543:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576142.101548:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576142.101549:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576142.101552:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576142.101554:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576142.103743:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576142.103747:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576142.103751:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576142.231164:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576142.231175:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576142.231178:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576142.231186:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576142.231190:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576142.231195:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576142.231201:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576142.231204:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576142.231206:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576143.125808:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576143.125816:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576143.125824:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576143.125825:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576143.125831:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576143.125832:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576143.126315:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576143.126319:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576143.126321:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576143.253495:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576143.253503:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576143.253504:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576143.253508:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576143.253512:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576143.253515:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576143.253518:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576143.253520:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576143.253523:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576143.766013:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576143.766026:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576143.766029:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576143.766034:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576143.766038:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576143.766045:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13074
00000400:00000200:3.0:1788576143.766054:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576143.766061:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576143.766064:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576143.766067:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766070:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576143.766072:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766075:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576143.766078:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576143.766080:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576143.766083:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576143.766086:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576143.766088:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576143.766090:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576143.766094:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766096:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576143.766098:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766100:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576143.766103:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576143.766106:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766108:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576143.766111:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576143.766115:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576143.766117:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576143.766119:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576143.766122:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576143.766131:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000020:00000040:3.0:1788576143.766134:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576143.766138:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576143.766140:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576143.766143:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576143.766146:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576143.766150:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576143.766158:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfa400.
00000100:00000001:3.0:1788576143.766165:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576143.766174:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576143.766184:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576143.766188:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576143.766194:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.766256:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576143.766261:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576143.766263:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576143.766271:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.766279:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576143.766280:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576143.766283:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576143.766286:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576143.766289:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576143.766293:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393262336:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576143.766297:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576143.766301:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576143.766303:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576143.766306:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576143.766311:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576143.766313:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576143.766319:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576143.766323:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576143.766329:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576143.766336:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393262336, portal 25
00000100:00000001:2.0:1788576143.766338:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576143.766339:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:2.0:1788576143.766345:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576168 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.766352:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576143.766355:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576143.766358:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576143.766368:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393262336, offset 0
00000400:00000200:2.0:1788576143.766375:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576143.766383:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576143.766415:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576143.766421:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576143.766425:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576143.766429:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576143.766432:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576143.766436:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331305:331305:256:4294967295] 192.168.202.110@tcp LPNI seq info [331262:331262:8:4294967295]
00000400:00000200:2.0:1788576143.766453:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576143.766463:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576143.766469:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576143.766475:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576143.766481:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576143.766485:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788576143.766491:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576143.766494:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788576143.766498:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576143.766501:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576143.766507:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576143.766512:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576143.766518:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576143.766520:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576143.766524:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576143.766528:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576168 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576143.766595:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576143.766600:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576143.766603:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576143.766604:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576143.766606:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576143.766609:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576143.766612:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13074
00000400:00000200:3.0:1788576143.766616:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576143.766619:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576143.766622:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576143.766624:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766626:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576143.766628:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766630:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576143.766632:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576143.766633:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576143.766635:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576143.766637:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788576143.766639:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576143.766641:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788576143.766644:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766655:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576143.766657:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766658:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576143.766661:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788576143.766664:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576143.766667:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576143.766669:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576143.766674:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576143.766676:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576143.766678:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576143.766682:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788576143.766684:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576143.766686:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576143.766691:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788576143.766693:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788576143.766696:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576143.766697:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576143.766698:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576143.766700:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576143.766701:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576143.766705:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8000.
00000100:00000001:3.0:1788576143.766709:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576143.766714:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576143.766721:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576143.766723:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576143.766727:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767034:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576143.767040:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576168 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767047:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576143.767049:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576143.767056:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576143.767061:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576143.767063:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307748282
00000100:00000010:2.0:1788576143.767069:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c420.
00000100:00000001:2.0:1788576143.767075:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576143.767080:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307748282
00000100:00000010:2.0:1788576143.767083:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c9c0.
00000100:00000001:2.0:1788576143.767086:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576143.767090:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576143.767092:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576143.767094:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576143.767099:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576143/real 1788576143]  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576143.767108:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576143.767110:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576143.767112:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576143.767115:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576143.767117:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576143.767120:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576143.767123:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576143.767131:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767138:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576143.767139:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767141:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576143.767144:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576143.767147:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767154:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576143.767158:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767164:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576143.767167:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767169:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576143.767170:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767172:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576143.767174:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576143.767177:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576143.767180:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576143.767182:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576143.767185:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576143.767193:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393262336:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576143.767198:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767200:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576143.767203:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1c00 x1875444393262336/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767209:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576143.767211:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576143.767214:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576143.767216:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576143.767218:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576143.767219:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576143.767221:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576143.767224:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfa400.
02000000:00000001:2.0:1788576143.767227:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576143.767228:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576143.767230:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1c00.
00000100:00000001:2.0:1788576143.767233:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576143.767234:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767237:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767245:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576143.767246:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576143.767249:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576143.767250:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576143.767252:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576143.767255:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767261:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576143.767262:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576143.767264:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576143.767266:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576143.767268:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576143.767277:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262464:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576143.767280:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576143.767282:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576143.767284:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576143.767285:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576143.767288:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576143.767290:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576143.767294:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576143.767296:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576143.767300:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576143.767303:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393262464, portal 10
00000100:00000001:2.0:1788576143.767305:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576143.767306:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788576143.767312:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576168 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767319:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576143.767321:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576143.767323:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576143.767326:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393262464, offset 0
00000400:00000200:2.0:1788576143.767330:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576143.767334:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576143.767339:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576143.767343:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576143.767345:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576143.767348:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576143.767350:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576143.767353:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331306:331306:256:4294967295] 192.168.202.110@tcp LPNI seq info [331263:331263:8:4294967295]
00000400:00000200:2.0:1788576143.767357:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576143.767493:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576143.767497:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576143.767500:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788576143.767501:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576143.767503:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576143.767506:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576143.767509:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13074
00000400:00000200:3.0:1788576143.767513:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576143.767515:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576143.767517:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576143.767520:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.767522:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576143.767524:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.767526:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576143.767528:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576143.767530:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576143.767532:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576143.767534:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788576143.767536:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788576143.767538:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788576143.767541:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.767543:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576143.767545:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576143.767546:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576143.767548:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788576143.767551:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576143.767554:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576143.767555:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576143.767559:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576143.767561:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576143.767562:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576143.767564:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788576143.767566:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576143.767568:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576143.767572:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000020:00000040:3.0:1788576143.767575:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788576143.767578:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576143.767579:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576143.767580:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576143.767582:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576143.767584:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576143.767588:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9400.
00000100:00000001:3.0:1788576143.767591:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576143.767597:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576143.767605:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576143.767608:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576143.767612:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576143.767619:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576143.767622:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788576143.767627:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788576143.767631:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13085)
00000400:00000200:2.0:1788576143.767678:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576143.767685:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576143.767690:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576143.767694:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576143.767696:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788576143.767700:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576143.767702:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788576143.767706:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576143.767708:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576143.767711:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576143.767715:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576143.767719:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576143.767722:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576143.767725:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576143.767728:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576168 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767799:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576143.767803:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576168 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767809:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576143.767811:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576143.767815:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576143.767819:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576143.767823:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576143.767825:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307747368
00000100:00000001:2.0:1788576143.767827:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307748282
00000100:00000001:2.0:1788576143.767829:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307747368
00000100:00000001:2.0:1788576143.767831:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307748282
00000100:00000001:2.0:1788576143.767833:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576143.767835:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576143.767837:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576143.767841:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576143/real 1788576143]  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576143.767848:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576143.767849:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576143.767851:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576143.767853:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576143.767854:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576143.767857:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576143.767860:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576143.767867:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767874:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576143.767875:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767877:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576143.767879:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576143.767882:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767888:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576143.767892:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767898:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576143.767900:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767903:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576143.767904:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767906:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576143.767907:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576143.767910:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576143.767914:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576143.767916:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576143.767920:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576143.767928:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262464:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576143.767932:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767933:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576143.767936:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0380 x1875444393262464/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.767942:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576143.767944:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576143.767947:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576143.767950:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576143.767952:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576143.767953:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576143.767955:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576143.767956:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8000.
02000000:00000001:2.0:1788576143.767959:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576143.767980:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576143.767982:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0380.
00000100:00000001:2.0:1788576143.767985:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576143.767986:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767988:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.767991:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576143.767992:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576143.767994:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576143.767995:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576143.767997:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576143.768001:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.768008:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576143.768010:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576143.768012:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576143.768013:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576143.768015:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576143.768018:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262592:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576143.768021:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576143.768023:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576143.768024:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576143.768026:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576143.768029:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576143.768032:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576143.768036:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576143.768038:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576143.768042:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576143.768045:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393262592, portal 4
00000100:00000001:2.0:1788576143.768047:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576143.768048:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788576143.768054:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576168 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.768060:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576143.768063:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576143.768065:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576143.768067:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393262592, offset 0
00000400:00000200:2.0:1788576143.768071:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576143.768075:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576143.768078:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576143.768082:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576143.768085:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576143.768087:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576143.768090:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576143.768093:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331307:331307:256:4294967295] 192.168.202.110@tcp LPNI seq info [331264:331264:8:4294967295]
00000400:00000200:2.0:1788576143.768097:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576143.768102:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576143.768106:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576143.768110:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576143.768114:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576143.768117:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1788576143.768121:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576143.768123:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1788576143.768127:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576143.768128:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576143.768131:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576143.768134:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576143.768137:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576143.768139:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576143.768142:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576143.768145:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576168 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.768152:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576143.768155:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576168 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.768161:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576143.768163:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576143.768166:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576143.768170:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576143.768172:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576143.768174:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307747514
00000100:00000001:2.0:1788576143.768176:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307748282
00000100:00000001:2.0:1788576143.768178:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307747514
00000100:00000001:2.0:1788576143.768180:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307748282
00000100:00000001:2.0:1788576143.768182:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576143.768184:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576143.768186:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576143.768189:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576143/real 1788576143]  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576143.768198:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576143.768199:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576143.768201:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576143.768203:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576143.768205:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576143.768206:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576143.768209:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576143.768216:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.768222:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576143.768223:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.768225:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576143.768226:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576143.768230:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.768236:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576143.768239:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.768246:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576143.768248:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.768250:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576143.768251:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.768253:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576143.768255:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576143.768258:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576143.768262:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576143.768264:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576143.768268:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576143.768276:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262592:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576143.768279:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.768281:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576143.768284:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393262592/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576168 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576143.768290:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576143.768292:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576143.768293:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576143.768295:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576143.768297:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576143.768298:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576143.768300:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576143.768301:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9400.
02000000:00000001:2.0:1788576143.768304:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576143.768305:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576143.768306:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788576143.768309:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576143.768310:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.768312:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576143.768315:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576143.768316:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576143.768318:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576143.768320:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576144.149616:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576144.149625:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576144.149631:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576144.149633:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576144.149636:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576144.149637:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576144.149999:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576144.150004:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576144.150008:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576144.279095:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576144.279106:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576144.279109:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576144.279117:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576144.279121:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576144.279127:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576144.279130:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576144.279133:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576144.279136:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576144.725592:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307748282, at 4307748288
00000100:00000010:1.0:1788576144.725616:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c9c0.
00000100:00000040:0.0:1788576144.726292:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307748282, at 4307748288
00000100:00000010:0.0:1788576144.726302:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c420.
00010000:00000001:3.0:1788576145.174985:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576145.174992:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576145.174997:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576145.174999:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576145.175002:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576145.175003:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576145.175639:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576145.175644:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576145.175647:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576145.301616:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576145.301626:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576145.301629:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576145.301636:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576145.301641:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576145.301646:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576145.301650:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576145.301653:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576145.301656:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576146.199388:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576146.199413:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576146.199421:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576146.199422:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576146.199426:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576146.199427:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576146.201622:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576146.201629:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576146.201635:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576146.325517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576146.325529:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576146.325532:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576146.325540:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576146.325547:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576146.325551:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576146.325555:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576146.325558:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576146.325562:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576147.221508:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576147.221513:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576147.221518:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576147.221521:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576147.221523:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576147.221527:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576147.221530:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576147.221532:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576147.221534:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576147.221536:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576147.221540:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576147.221542:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576147.221544:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576147.221545:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576147.221546:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576147.221547:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576147.221549:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576147.221550:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576147.221678:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576147.221681:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576147.221684:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576147.349838:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576147.349848:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576147.349850:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576147.349857:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576147.349861:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576147.349866:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576147.349869:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576147.349871:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576147.349874:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576148.245515:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576148.245522:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576148.245528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576148.245529:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576148.245532:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576148.245533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576148.245684:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576148.245688:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576148.245691:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576148.376034:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576148.376042:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576148.376044:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576148.376050:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576148.376053:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576148.376057:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576148.376059:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576148.376061:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576148.376064:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576148.885746:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576148.885754:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576148.885756:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576148.885760:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576148.885763:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576148.885769:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13080
00000400:00000200:3.0:1788576148.885775:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576148.885789:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576148.885791:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576148.885793:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885795:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576148.885797:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885799:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576148.885801:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576148.885803:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576148.885805:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576148.885807:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576148.885808:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576148.885810:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576148.885812:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885813:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576148.885815:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885816:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576148.885818:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576148.885820:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885821:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576148.885823:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576148.885826:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576148.885828:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576148.885829:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576148.885830:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576148.885838:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e2300.
00000020:00000040:3.0:1788576148.885841:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576148.885843:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576148.885844:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576148.885847:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576148.885849:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576148.885852:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576148.885858:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf9800.
00000100:00000001:3.0:1788576148.885863:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576148.885869:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576148.885876:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576148.885880:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576148.885883:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc4bce75] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576148.885901:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576148.885903:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576148.885905:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576148.885906:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576148.885907:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576148.885909:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576148.885911:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13080
00000400:00000200:3.0:1788576148.885914:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576148.885916:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576148.885917:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576148.885919:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885920:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576148.885922:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885923:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576148.885924:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576148.885925:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576148.885926:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576148.885928:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788576148.885929:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576148.885931:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788576148.885933:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885934:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576148.885935:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885936:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000001:2.0:1788576148.885937:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1788576148.885938:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788576148.885940:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576148.885943:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576148.885944:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
02000000:00000001:3.0:1788576148.885944:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1788576148.885948:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1788576148.885950:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576148.885953:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576148.885956:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1788576148.885958:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1788576148.885959:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788576148.885964:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576148.885968:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576148.885972:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0a80.
00000100:00000001:2.0:1788576148.885998:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:3.0:1788576148.885999:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
00000100:00000001:2.0:1788576148.886002:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576148.886003:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576148.886006:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576148.886007:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1788576148.886008:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576148.886010:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576148.886011:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576148.886013:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576148.886017:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576148.886018:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb400.
00000100:00000001:3.0:1788576148.886023:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576148.886024:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393262720:192.168.202.110@tcp:250:kworker.0
00000100:00100000:3.0:1788576148.886029:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576148.886031:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576148.886036:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1788576148.886037:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
02000000:00000001:2.0:1788576148.886041:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576148.886041:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576148.886047:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:3.0:1788576148.886047:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b298c26f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788576148.886053:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
00000100:00000001:3.0:1788576148.886054:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576148.886058:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576148.886058:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576148.886063:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788576148.886064:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576148.886065:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1788576148.886068:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576148.886069:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576148.886073:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13080
00000400:00000010:2.0:1788576148.886075:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000200:3.0:1788576148.886076:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576148.886080:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576148.886082:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576148.886084:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.886085:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576148.886086:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.886088:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576148.886089:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576148.886090:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576148.886091:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576148.886092:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788576148.886094:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000400:00000010:2.0:1788576148.886096:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000020:00000001:3.0:1788576148.886096:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788576148.886099:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.886101:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576148.886103:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576148.886104:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000100:00000200:2.0:1788576148.886105:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393262720, portal 25
00000020:00000040:3.0:1788576148.886106:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00000001:2.0:1788576148.886110:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1788576148.886110:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576148.886114:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138823424 : -130483570728192 : ffff89536d1e2300)
00000100:00000001:3.0:1788576148.886115:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576148.886117:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576148.886121:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576148.886124:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576148.886126:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576173 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576148.886126:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576148.886129:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788576148.886132:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576148.886135:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576148.886140:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788576148.886141:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:3.0:1788576148.886143:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
00000100:00000040:2.0:1788576148.886148:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:3.0:1788576148.886148:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576148.886151:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1788576148.886153:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000001:3.0:1788576148.886153:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576148.886155:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576148.886158:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1788576148.886159:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393262720, offset 0
02000000:00000010:3.0:1788576148.886161:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8800.
00000100:00000001:3.0:1788576148.886164:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576148.886168:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00100000:3.0:1788576148.886169:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576148.886175:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576148.886178:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576148.886179:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1788576148.886183:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005320734a] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788576148.886188:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1788576148.886192:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576148.886195:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1788576148.886196:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1788576148.886198:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1788576148.886202:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1788576148.886203:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13090)
00000400:00000200:2.0:1788576148.886209:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576148.886212:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576148.886218:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331308:331308:256:4294967295] 192.168.202.110@tcp LPNI seq info [331265:331265:8:4294967295]
00000400:00000200:2.0:1788576148.886223:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576148.886232:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576148.886238:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576148.886242:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576148.886248:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576148.886252:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788576148.886257:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576148.886260:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788576148.886264:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576148.886267:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576148.886273:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576148.886277:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576148.886282:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576148.886284:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576148.886288:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576148.886292:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576173 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886302:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576148.886305:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576173 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886313:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576148.886314:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576148.886320:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576148.886323:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576148.886325:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307753401
00000100:00000010:2.0:1788576148.886330:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c0c0.
00000100:00000001:2.0:1788576148.886336:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576148.886341:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307753401
00000100:00000010:2.0:1788576148.886343:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0ca20.
00000100:00000001:2.0:1788576148.886346:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576148.886350:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576148.886353:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576148.886355:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576148.886359:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576148/real 1788576148]  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576148.886368:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576148.886370:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576148.886372:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576148.886374:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576148.886376:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576148.886378:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576148.886381:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576148.886389:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886415:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576148.886417:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.886419:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576148.886421:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576148.886425:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886432:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576148.886436:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886443:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576148.886446:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.886447:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576148.886448:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.886450:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576148.886453:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576148.886456:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576148.886468:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576148.886471:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576148.886476:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576148.886484:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393262720:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576148.886489:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.886491:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576148.886494:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e2300 x1875444393262720/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886500:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576148.886502:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576148.886504:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788576148.886507:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576148.886509:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576148.886510:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576148.886513:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576148.886515:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf9800.
02000000:00000001:2.0:1788576148.886518:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576148.886520:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576148.886521:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e2300.
00000100:00000001:2.0:1788576148.886524:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576148.886525:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.886527:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.886535:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576148.886537:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576148.886539:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576148.886540:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576148.886542:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576148.886545:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886552:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576148.886553:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576148.886555:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576148.886556:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576148.886559:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576148.886562:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576148.886565:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576148.886567:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576148.886568:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576148.886570:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576148.886573:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788576148.886575:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576148.886578:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576148.886581:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576148.886584:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576148.886587:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393262848, portal 10
00000100:00000001:2.0:1788576148.886589:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576148.886591:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138817152 : -130483570734464 : ffff89536d1e0a80)
00000100:00000040:2.0:1788576148.886596:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576173 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886602:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576148.886604:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576148.886606:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576148.886608:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393262848, offset 0
00000400:00000200:2.0:1788576148.886611:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576148.886615:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576148.886619:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576148.886621:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576148.886623:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576148.886626:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576148.886628:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576148.886631:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331309:331309:256:4294967295] 192.168.202.110@tcp LPNI seq info [331266:331266:8:4294967295]
00000400:00000200:2.0:1788576148.886635:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576148.886642:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576148.886646:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576148.886649:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576148.886652:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576148.886655:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788576148.886659:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576148.886662:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788576148.886665:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576148.886667:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576148.886669:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576148.886673:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576148.886675:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576148.886677:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576148.886679:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576148.886682:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576173 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886689:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576148.886693:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576173 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886699:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576148.886700:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576148.886702:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576148.886706:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576148.886709:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576148.886711:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307752487
00000100:00000001:2.0:1788576148.886714:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307753401
00000100:00000001:2.0:1788576148.886716:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307752487
00000100:00000001:2.0:1788576148.886718:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307753401
00000100:00000001:2.0:1788576148.886720:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576148.886722:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576148.886723:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576148.886727:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576148/real 1788576148]  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576148.886741:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576148.886742:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576148.886744:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576148.886746:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576148.886748:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576148.886749:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576148.886753:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576148.886761:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886767:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576148.886768:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.886770:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576148.886771:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576148.886775:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886781:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576148.886785:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886792:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576148.886794:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.886796:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576148.886797:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.886798:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576148.886800:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576148.886802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576148.886805:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576148.886807:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576148.886812:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576148.886819:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576148.886822:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576148.886826:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886832:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576148.886833:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576148.886835:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576148.886837:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576148.886838:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576148.886841:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576148.886843:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576148.886845:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576148.886847:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576148.886848:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576148.886851:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576148.886853:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576148.886856:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576148.886858:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576148.886861:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576148.886864:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393262976, portal 4
00000100:00000001:2.0:1788576148.886866:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576148.886867:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138820736 : -130483570730880 : ffff89536d1e1880)
00000100:00000040:2.0:1788576148.886872:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576173 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886878:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576148.886880:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576148.886882:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576148.886885:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393262976, offset 0
00000400:00000200:2.0:1788576148.886888:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576148.886891:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576148.886894:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576148.886897:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576148.886899:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576148.886902:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576148.886904:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576148.886907:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331310:331310:256:4294967295] 192.168.202.110@tcp LPNI seq info [331267:331267:8:4294967295]
00000400:00000200:2.0:1788576148.886911:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576148.886917:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576148.886921:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576148.886924:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576148.886927:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576148.886930:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1788576148.886934:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576148.886937:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1788576148.886941:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576148.886942:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576148.886945:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576148.886948:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576148.886951:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576148.886952:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576148.886954:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576148.886958:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576173 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886965:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576148.886973:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576173 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.886990:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576148.886992:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576148.886995:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576148.886997:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576148.887000:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576148.887002:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307752633
00000100:00000001:2.0:1788576148.887005:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307753401
00000100:00000001:2.0:1788576148.887007:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307752633
00000100:00000001:2.0:1788576148.887009:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307753401
00000100:00000001:2.0:1788576148.887011:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576148.887013:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576148.887014:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576148.887018:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576148/real 1788576148]  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576148.887025:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576148.887027:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576148.887028:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576148.887030:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576148.887032:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576148.887033:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576148.887036:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576148.887044:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.887050:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576148.887051:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.887053:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576148.887054:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576148.887058:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.887064:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576148.887068:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.887074:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576148.887076:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.887078:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576148.887079:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.887081:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576148.887082:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576148.887085:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576148.887088:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576148.887090:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576148.887093:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576148.887102:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff89536d1e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393262976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576148.887105:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.887107:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576148.887110:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e0a80 x1875444393262848/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.887116:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576148.887117:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576148.887119:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12000.
02000000:00000001:2.0:1788576148.887122:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576148.887123:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576148.887124:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576148.887126:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576148.887128:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cfb400.
02000000:00000001:2.0:1788576148.887131:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576148.887132:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576148.887133:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e0a80.
00000100:00000001:2.0:1788576148.887135:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576148.887136:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.887138:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576148.887141:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff89536d1e1880 x1875444393262976/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576173 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576148.887153:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576148.887154:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576148.887155:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f11400.
02000000:00000001:2.0:1788576148.887157:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576148.887158:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576148.887159:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576148.887161:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576148.887163:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cf8800.
02000000:00000001:2.0:1788576148.887165:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576148.887166:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576148.887167:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff89536d1e1880.
00000100:00000001:2.0:1788576148.887169:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576148.887170:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.887172:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576148.887175:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576148.887176:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576148.887177:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576148.887179:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576149.269529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576149.269537:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576149.269539:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576149.269543:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576149.269547:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576149.269549:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576149.269551:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576149.269554:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576149.269558:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576149.269559:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576149.269563:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576149.269564:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576149.269791:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576149.269795:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576149.269797:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576149.398358:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576149.398367:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576149.398369:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576149.398375:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576149.398378:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576149.398382:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576149.398384:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576149.398387:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576149.398389:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576149.845525:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307753401, at 4307753408
00000100:00000010:0.0:1788576149.845541:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c0c0.
00000100:00000040:1.0:1788576149.845664:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307753401, at 4307753408
00000100:00000010:1.0:1788576149.845671:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0ca20.
00010000:00000001:3.0:1788576150.293528:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576150.293535:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576150.293541:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576150.293542:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576150.293546:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576150.293547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576150.293623:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576150.293626:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576150.293628:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576150.422260:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576150.422267:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576150.422269:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576150.422274:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576150.422278:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576150.422281:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576150.422284:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576150.422286:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576150.422289:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576151.317819:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576151.317826:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576151.317831:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576151.317832:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576151.317835:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576151.317836:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576151.319055:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576151.319061:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576151.319067:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576151.448884:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576151.448895:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576151.448897:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576151.448905:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576151.448909:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576151.448913:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576151.448916:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576151.448932:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576151.448936:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576152.341837:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576152.341846:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576152.341853:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576152.341854:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576152.341858:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576152.341860:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576152.341966:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576152.341970:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576152.341974:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576152.469574:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576152.469584:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576152.469586:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576152.469594:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576152.469598:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576152.469602:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576152.469606:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576152.469610:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576152.469614:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576153.366110:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576153.366119:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576153.366126:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576153.366128:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576153.366132:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576153.366133:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576153.366306:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576153.366310:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576153.366314:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576153.493901:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576153.493910:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576153.493912:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576153.493920:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576153.493925:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576153.493930:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576153.493932:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576153.493935:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576153.493938:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576154.005552:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576154.005568:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788576154.005572:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576154.005577:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576154.005582:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576154.005592:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13085
00000400:00000200:0.0:1788576154.005604:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576154.005613:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576154.005616:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576154.005620:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005624:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576154.005627:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005629:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576154.005634:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576154.005636:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788576154.005639:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576154.005643:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788576154.005645:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788576154.005648:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788576154.005651:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005653:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576154.005654:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005656:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576154.005660:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788576154.005664:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005667:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576154.005671:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576154.005675:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576154.005679:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576154.005681:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576154.005684:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576154.005697:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712300.
00000020:00000040:0.0:1788576154.005701:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788576154.005705:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576154.005707:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576154.005711:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576154.005715:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576154.005719:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576154.005731:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4400.
00000100:00000001:0.0:1788576154.005751:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576154.005763:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576154.005779:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576154.005785:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576154.005790:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098fa8172] to pc [ptlrpcd_rcv+-1]  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576154.005826:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576154.005830:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576154.005834:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788576154.005835:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576154.005837:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576154.005840:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576154.005843:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13085
00000400:00000200:0.0:1788576154.005847:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576154.005851:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576154.005853:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576154.005855:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005858:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576154.005860:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005862:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576154.005864:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576154.005865:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788576154.005867:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576154.005869:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788576154.005871:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788576154.005873:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788576154.005876:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005878:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576154.005880:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005881:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576154.005894:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788576154.005898:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576154.005900:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576154.005902:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576154.005910:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576154.005912:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576154.005915:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576154.005919:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788576154.005922:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576154.005924:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576154.005928:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710e00.
00000020:00000040:0.0:1788576154.005931:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788576154.005933:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576154.005934:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576154.005936:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576154.005937:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576154.005939:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576154.005943:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6000.
00000100:00000001:0.0:1788576154.005947:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576154.005953:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576154.005970:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576154.005972:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576154.005976:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008a58a26] to pc [ptlrpcd_rcv+-1]  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576154.005983:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576154.005985:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576154.006003:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576154.006004:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576154.006006:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576154.006009:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576154.006012:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13085
00000400:00000200:0.0:1788576154.006015:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576154.006018:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576154.006021:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576154.006023:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.006025:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576154.006027:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.006029:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576154.006031:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576154.006033:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788576154.006045:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576154.006048:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788576154.006050:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788576154.006052:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788576154.006055:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.006057:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576154.006059:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576154.006062:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576154.006064:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788576154.006067:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576154.006069:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576154.006071:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576154.006074:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576154.006077:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576154.006078:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576154.006081:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788576154.006083:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576154.006084:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576154.006088:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712a00.
00000020:00000040:0.0:1788576154.006091:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788576154.006093:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576154.006094:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576154.006096:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576154.006098:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576154.006099:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576154.006103:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd7400.
00000100:00000001:0.0:1788576154.006107:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576154.006111:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576154.006118:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576154.006120:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576154.006123:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000814f71c3] to pc [ptlrpcd_rcv+-1]  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576154.006130:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576154.006133:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788576154.006137:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788576154.006143:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13095)
00000100:00000001:2.0:1788576154.006350:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576154.006357:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576154.006361:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576154.006369:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.006379:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576154.006382:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576154.006386:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576154.006389:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576154.006415:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576154.006420:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393263104:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576154.006425:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576154.006431:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576154.006434:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576154.006437:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576154.006443:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576154.006446:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576154.006453:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576154.006458:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576154.006466:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576154.006474:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393263104, portal 25
00000100:00000001:2.0:1788576154.006477:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576154.006479:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523505920 : -130484186045696 : ffff895348712300)
00000100:00000040:2.0:1788576154.006486:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.006494:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576154.006498:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576154.006501:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576154.006505:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393263104, offset 0
00000400:00000200:2.0:1788576154.006512:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576154.006521:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576154.006528:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576154.006535:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576154.006540:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576154.006544:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576154.006547:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576154.006552:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331311:331311:256:4294967295] 192.168.202.110@tcp LPNI seq info [331268:331268:8:4294967295]
00000400:00000200:2.0:1788576154.006557:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576154.006568:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576154.006574:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576154.006580:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576154.006588:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576154.006593:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788576154.006600:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576154.006603:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788576154.006608:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576154.006611:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576154.006619:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576154.006625:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576154.006633:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576154.006636:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576154.006642:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576154.006646:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.006658:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576154.006661:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576179 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.006669:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576154.006672:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576154.006678:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576154.006683:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576154.006686:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307758522
00000100:00000010:2.0:1788576154.006692:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c480.
00000100:00000001:2.0:1788576154.006700:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576154.006707:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307758522
00000100:00000010:2.0:1788576154.006710:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0cd80.
00000100:00000001:2.0:1788576154.006714:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576154.006718:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576154.006721:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576154.006724:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576154.006728:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576154/real 1788576154]  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576154.006739:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576154.006741:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576154.006743:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576154.006746:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576154.006748:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576154.006750:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576154.006754:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576154.006763:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.006770:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576154.006772:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.006775:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576154.006777:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576154.006782:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.006789:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576154.006794:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.006801:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576154.006804:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.006806:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576154.006807:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.006810:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576154.006812:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576154.006815:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576154.006819:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576154.006821:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576154.006826:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576154.006834:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393263104:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576154.006839:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576154.006843:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.006849:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576154.006851:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576154.006853:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576154.006855:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576154.006857:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576154.006860:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393263232:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576154.006864:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576154.006866:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576154.006868:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576154.006869:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576154.006872:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788576154.006874:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576154.006878:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576154.006880:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576154.006883:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576154.006886:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393263232, portal 10
00000100:00000001:2.0:1788576154.006888:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576154.006889:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523500544 : -130484186051072 : ffff895348710e00)
00000100:00000040:2.0:1788576154.006896:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.006902:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576154.006904:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576154.006906:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576154.006908:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393263232, offset 0
00000400:00000200:2.0:1788576154.006912:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576154.006915:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576154.006919:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576154.006922:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576154.006924:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576154.006926:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576154.006929:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576154.006931:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331312:331312:256:4294967295] 192.168.202.110@tcp LPNI seq info [331269:331269:8:4294967295]
00000400:00000200:2.0:1788576154.006936:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576154.006941:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576154.006945:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576154.006948:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576154.006952:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576154.006954:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788576154.006959:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576154.006961:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788576154.006965:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576154.006966:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576154.006969:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576154.006972:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576154.006975:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576154.006976:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576154.006979:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576154.006982:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007019:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576154.007022:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576179 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007029:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576154.007031:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576154.007035:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576154.007039:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576154.007042:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576154.007044:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307757607
00000100:00000001:2.0:1788576154.007047:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307758522
00000100:00000001:2.0:1788576154.007049:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307757607
00000100:00000001:2.0:1788576154.007051:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307758522
00000100:00000001:2.0:1788576154.007053:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576154.007055:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576154.007056:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576154.007060:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576154/real 1788576154]  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576154.007075:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576154.007076:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576154.007078:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576154.007080:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576154.007081:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576154.007084:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576154.007087:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576154.007095:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007101:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576154.007103:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007105:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576154.007106:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576154.007110:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007117:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576154.007121:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007127:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576154.007129:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007131:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576154.007132:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007134:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576154.007136:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576154.007138:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576154.007141:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576154.007144:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576154.007148:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576154.007156:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393263232:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576154.007160:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576154.007164:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007169:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576154.007171:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576154.007173:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576154.007175:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576154.007177:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576154.007180:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393263360:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576154.007183:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576154.007186:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576154.007187:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576154.007189:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576154.007193:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788576154.007195:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576154.007199:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576154.007202:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576154.007205:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576154.007208:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393263360, portal 4
00000100:00000001:2.0:1788576154.007211:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576154.007212:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523507712 : -130484186043904 : ffff895348712a00)
00000100:00000040:2.0:1788576154.007219:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007225:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576154.007227:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576154.007229:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576154.007232:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393263360, offset 0
00000400:00000200:2.0:1788576154.007236:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576154.007240:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576154.007244:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576154.007247:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576154.007249:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576154.007252:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576154.007254:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576154.007257:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331313:331313:256:4294967295] 192.168.202.110@tcp LPNI seq info [331270:331270:8:4294967295]
00000400:00000200:2.0:1788576154.007261:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576154.007267:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576154.007272:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576154.007275:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576154.007279:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1788576154.007282:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1788576154.007286:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576154.007288:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1788576154.007292:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576154.007294:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534b133200.
00000400:00000200:2.0:1788576154.007297:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1788576154.007301:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1788576154.007304:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c657f8
00000400:00000010:2.0:1788576154.007305:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c657f8.
00000100:00000001:2.0:1788576154.007308:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:2.0:1788576154.007311:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576179 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007318:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576154.007320:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576179 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007326:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576154.007328:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:2.0:1788576154.007331:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576154.007333:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576154.007337:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576154.007339:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307757753
00000100:00000001:2.0:1788576154.007342:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307758522
00000100:00000001:2.0:1788576154.007344:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307757753
00000100:00000001:2.0:1788576154.007346:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307758522
00000100:00000001:2.0:1788576154.007348:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576154.007350:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576154.007351:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1788576154.007355:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576154/real 1788576154]  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1788576154.007363:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1788576154.007365:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1788576154.007366:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89535119fb00.
00000400:00000200:2.0:1788576154.007369:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff895343c656e8
00000400:00000010:2.0:1788576154.007370:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff895343c656e8.
00000100:00000001:2.0:1788576154.007372:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:2.0:1788576154.007375:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788576154.007383:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007389:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:2.0:1788576154.007422:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007425:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576154.007427:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1788576154.007431:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007437:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1788576154.007441:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007447:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1788576154.007450:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007452:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1788576154.007453:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007454:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1788576154.007456:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1788576154.007459:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1788576154.007462:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1788576154.007464:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1788576154.007468:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1788576154.007476:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393263360:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576154.007480:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007483:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576154.007486:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712300 x1875444393263104/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007492:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576154.007495:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576154.007498:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10800.
02000000:00000001:2.0:1788576154.007501:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576154.007503:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576154.007505:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1788576154.007507:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576154.007510:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4400.
02000000:00000001:2.0:1788576154.007515:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576154.007517:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576154.007518:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712300.
00000100:00000001:2.0:1788576154.007522:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576154.007523:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007525:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576154.007528:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710e00 x1875444393263232/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007539:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576154.007541:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576154.007542:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f10000.
02000000:00000001:2.0:1788576154.007544:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576154.007545:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576154.007546:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:2.0:1788576154.007548:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576154.007549:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6000.
02000000:00000001:2.0:1788576154.007551:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576154.007553:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576154.007554:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710e00.
00000100:00000001:2.0:1788576154.007556:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576154.007557:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007559:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1788576154.007563:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712a00 x1875444393263360/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576179 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576154.007569:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1788576154.007570:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1788576154.007571:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895349f12800.
02000000:00000001:2.0:1788576154.007574:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1788576154.007575:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1788576154.007576:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:2.0:1788576154.007578:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1788576154.007580:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd7400.
02000000:00000001:2.0:1788576154.007582:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1788576154.007583:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1788576154.007584:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712a00.
00000100:00000001:2.0:1788576154.007586:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1788576154.007588:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007590:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576154.007600:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576154.007603:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576154.007605:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576154.007607:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576154.389575:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576154.389585:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576154.389594:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576154.389609:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576154.389615:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576154.389616:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576154.391143:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576154.391152:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576154.391158:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576154.520425:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576154.520436:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576154.520439:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576154.520447:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576154.520451:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576154.520457:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576154.520460:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576154.520464:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576154.520467:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576154.965837:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307758522, at 4307758528
00000100:00000010:0.0:1788576154.965858:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0c480.
00000100:00000040:1.0:1788576154.967117:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307758522, at 4307758529
00000100:00000010:1.0:1788576154.967127:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff895344f0cd80.
00010000:00000001:3.0:1788576155.414125:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576155.414134:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576155.414141:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576155.414143:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576155.414147:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576155.414149:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576155.414340:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576155.414345:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576155.414348:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576155.542645:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576155.542656:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576155.542659:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576155.542667:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576155.542672:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576155.542678:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576155.542681:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576155.542684:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576155.542688:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576156.437573:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576156.437583:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576156.437591:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576156.437592:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576156.437596:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576156.437597:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576156.437735:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576156.437739:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576156.437743:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576156.568202:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576156.568213:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576156.568216:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576156.568225:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576156.568229:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576156.568234:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576156.568238:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576156.568241:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576156.568246:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576157.462124:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576157.462131:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576157.462136:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576157.462139:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576157.462141:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576157.462145:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576157.462148:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576157.462150:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576157.462152:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576157.462154:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576157.462158:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576157.462160:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576157.462161:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576157.462162:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576157.462164:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576157.462165:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576157.462166:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576157.462168:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576157.462274:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576157.462281:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576157.462286:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576157.589538:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576157.589545:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576157.589548:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576157.589553:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576157.589556:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576157.589561:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576157.589564:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576157.589566:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576157.589569:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576158.485523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576158.485529:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576158.485534:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576158.485536:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576158.485539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576158.485540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576158.485678:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576158.485682:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576158.485685:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576158.614199:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576158.614207:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576158.614209:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576158.614214:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576158.614219:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576158.614224:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576158.614227:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576158.614230:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576158.614235:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576159.125531:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576159.125542:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1788576159.125544:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576159.125548:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576159.125552:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576159.125559:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13090
00000400:00000200:0.0:1788576159.125569:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576159.125575:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576159.125578:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576159.125581:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125585:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576159.125587:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125589:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576159.125594:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576159.125596:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:0.0:1788576159.125599:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576159.125603:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:0.0:1788576159.125605:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:0.0:1788576159.125616:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:0.0:1788576159.125620:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125622:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576159.125623:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125625:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576159.125628:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:0.0:1788576159.125631:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125633:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576159.125636:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576159.125640:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576159.125643:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576159.125645:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1788576159.125648:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576159.125656:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348712680.
00000020:00000040:0.0:1788576159.125660:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1788576159.125663:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576159.125665:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576159.125667:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576159.125670:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576159.125675:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576159.125682:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd6800.
00000100:00000001:0.0:1788576159.125688:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576159.125697:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576159.125709:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576159.125713:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576159.125718:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001fbf2e9] to pc [ptlrpcd_rcv+-1]  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576159.125745:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576159.125748:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576159.125752:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1788576159.125753:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576159.125755:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576159.125758:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576159.125761:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13090
00000400:00000200:0.0:1788576159.125766:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576159.125769:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576159.125771:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576159.125774:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125776:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576159.125778:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125780:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576159.125783:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576159.125784:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:0.0:1788576159.125786:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576159.125788:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:0.0:1788576159.125790:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:0.0:1788576159.125792:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:0.0:1788576159.125795:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125797:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576159.125799:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125801:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576159.125803:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:0.0:1788576159.125806:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125809:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576159.125811:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576159.125816:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576159.125818:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576159.125820:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576159.125824:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1788576159.125826:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576159.125829:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576159.125832:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348713480.
00000020:00000040:0.0:1788576159.125834:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:0.0:1788576159.125837:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576159.125838:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576159.125840:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576159.125841:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576159.125843:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576159.125846:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd5400.
00000100:00000001:0.0:1788576159.125850:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576159.125855:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576159.125863:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576159.125865:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576159.125868:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ca99e3ff] to pc [ptlrpcd_rcv+-1]  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576159.125876:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1788576159.125879:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1788576159.125882:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1788576159.125884:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1788576159.125885:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1788576159.125888:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:0.0:1788576159.125890:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13090
00000400:00000200:0.0:1788576159.125894:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1788576159.125897:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1788576159.125899:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1788576159.125902:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125904:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576159.125906:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125909:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:0.0:1788576159.125911:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1788576159.125912:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:0.0:1788576159.125914:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1788576159.125916:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:0.0:1788576159.125918:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:0.0:1788576159.125921:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:0.0:1788576159.125924:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125926:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1788576159.125927:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125929:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:0.0:1788576159.125932:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:0.0:1788576159.125935:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1788576159.125937:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1788576159.125939:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1788576159.125941:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1788576159.125943:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576159.125945:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1788576159.125947:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:0.0:1788576159.125950:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576159.125952:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1788576159.125955:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348710380.
00000020:00000040:0.0:1788576159.125957:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:0.0:1788576159.125959:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1788576159.125961:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1788576159.125962:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1788576159.125964:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1788576159.125965:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1788576159.125969:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cd4000.
00000100:00000001:0.0:1788576159.125972:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576159.125976:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1788576159.125983:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1788576159.125985:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576159.125989:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cac5acb3] to pc [ptlrpcd_rcv+-1]  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1788576159.125997:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576159.126000:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1788576159.126014:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:0.0:1788576159.126019:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13100)
00000100:00000001:3.0:1788576159.126149:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788576159.126153:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788576159.126155:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788576159.126162:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126170:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576159.126172:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576159.126176:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576159.126178:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576159.126180:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576159.126184:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393263488:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788576159.126188:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576159.126193:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576159.126195:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576159.126198:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788576159.126202:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788576159.126205:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576159.126210:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788576159.126214:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788576159.126272:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788576159.126279:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393263488, portal 25
00000100:00000001:3.0:1788576159.126282:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576159.126284:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523506816 : -130484186044800 : ffff895348712680)
00000100:00000040:3.0:1788576159.126290:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126298:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788576159.126301:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788576159.126305:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788576159.126308:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393263488, offset 0
00000400:00000200:3.0:1788576159.126314:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576159.126322:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576159.126329:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788576159.126335:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788576159.126339:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788576159.126343:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788576159.126346:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788576159.126349:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331314:331314:256:4294967295] 192.168.202.110@tcp LPNI seq info [331271:331271:8:4294967295]
00000400:00000200:3.0:1788576159.126353:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576159.126362:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788576159.126367:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576159.126372:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788576159.126378:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788576159.126382:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788576159.126387:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788576159.126410:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788576159.126418:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576159.126421:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788576159.126427:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788576159.126432:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788576159.126438:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788576159.126440:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788576159.126445:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788576159.126449:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126458:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576159.126462:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576184 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126469:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576159.126471:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788576159.126478:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576159.126483:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576159.126486:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307763642
00000100:00000010:3.0:1788576159.126491:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d840.
00000100:00000001:3.0:1788576159.126498:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788576159.126504:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307763642
00000100:00000010:3.0:1788576159.126508:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91db40.
00000100:00000001:3.0:1788576159.126512:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788576159.126515:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788576159.126518:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576159.126521:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788576159.126526:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576159/real 1788576159]  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788576159.126535:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788576159.126537:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788576159.126539:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788576159.126542:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788576159.126544:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788576159.126546:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788576159.126550:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788576159.126558:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126565:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788576159.126567:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.126569:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576159.126571:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788576159.126576:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126583:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788576159.126587:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126596:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788576159.126598:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.126600:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576159.126601:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.126603:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788576159.126605:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788576159.126608:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788576159.126611:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788576159.126614:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788576159.126618:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788576159.126625:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393263488:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788576159.126629:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788576159.126632:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126639:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576159.126640:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576159.126643:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576159.126644:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576159.126646:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576159.126649:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393263616:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788576159.126652:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576159.126655:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576159.126656:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576159.126658:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788576159.126661:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788576159.126663:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576159.126667:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788576159.126670:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788576159.126673:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788576159.126676:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393263616, portal 10
00000100:00000001:3.0:1788576159.126678:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576159.126679:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523510400 : -130484186041216 : ffff895348713480)
00000100:00000040:3.0:1788576159.126686:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126692:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788576159.126695:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788576159.126697:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788576159.126699:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393263616, offset 0
00000400:00000200:3.0:1788576159.126703:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576159.126707:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576159.126711:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788576159.126714:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788576159.126716:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788576159.126719:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788576159.126721:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788576159.126724:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331315:331315:256:4294967295] 192.168.202.110@tcp LPNI seq info [331272:331272:8:4294967295]
00000400:00000200:3.0:1788576159.126729:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576159.126735:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788576159.126740:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576159.126743:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788576159.126746:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788576159.126749:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788576159.126754:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788576159.126756:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788576159.126760:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576159.126762:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788576159.126765:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788576159.126769:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788576159.126771:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788576159.126773:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788576159.126776:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788576159.126779:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126787:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576159.126790:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576184 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126796:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576159.126798:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788576159.126801:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576159.126805:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788576159.126808:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576159.126810:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307762727
00000100:00000001:3.0:1788576159.126813:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307763642
00000100:00000001:3.0:1788576159.126815:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307762727
00000100:00000001:3.0:1788576159.126817:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307763642
00000100:00000001:3.0:1788576159.126819:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788576159.126822:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576159.126824:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788576159.126827:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576159/real 1788576159]  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788576159.126835:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788576159.126837:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788576159.126839:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788576159.126841:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788576159.126843:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788576159.126845:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788576159.126848:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788576159.126857:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126870:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788576159.126871:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.126873:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576159.126875:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788576159.126879:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126885:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788576159.126889:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126896:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788576159.126898:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.126900:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576159.126901:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.126903:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788576159.126905:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788576159.126908:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788576159.126911:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788576159.126914:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788576159.126917:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788576159.126926:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393263616:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788576159.126930:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788576159.126933:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126940:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576159.126941:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576159.126944:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576159.126946:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576159.126948:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576159.126951:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393263744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788576159.126954:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576159.126956:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576159.126958:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576159.126959:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788576159.126962:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788576159.126964:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576159.126968:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788576159.126971:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788576159.126974:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788576159.126977:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393263744, portal 4
00000100:00000001:3.0:1788576159.126979:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576159.126980:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589523497856 : -130484186053760 : ffff895348710380)
00000100:00000040:3.0:1788576159.126985:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.126992:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788576159.126994:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788576159.126996:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788576159.126998:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393263744, offset 0
00000400:00000200:3.0:1788576159.127014:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576159.127018:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576159.127021:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788576159.127024:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788576159.127026:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788576159.127029:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788576159.127031:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788576159.127034:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331316:331316:256:4294967295] 192.168.202.110@tcp LPNI seq info [331273:331273:8:4294967295]
00000400:00000200:3.0:1788576159.127039:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576159.127045:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788576159.127050:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576159.127053:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788576159.127057:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788576159.127060:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1788576159.127064:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788576159.127066:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1788576159.127070:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576159.127072:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788576159.127075:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788576159.127078:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788576159.127081:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788576159.127083:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788576159.127085:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788576159.127089:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576184 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.127097:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576159.127099:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576184 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.127105:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576159.127107:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788576159.127110:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576159.127112:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788576159.127116:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576159.127118:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307762873
00000100:00000001:3.0:1788576159.127120:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307763642
00000100:00000001:3.0:1788576159.127122:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307762873
00000100:00000001:3.0:1788576159.127125:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307763642
00000100:00000001:3.0:1788576159.127127:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788576159.127129:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576159.127130:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788576159.127134:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576159/real 1788576159]  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788576159.127143:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788576159.127144:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788576159.127145:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788576159.127148:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788576159.127150:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788576159.127151:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788576159.127155:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788576159.127164:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.127170:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788576159.127171:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.127173:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576159.127174:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788576159.127178:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.127185:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788576159.127189:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.127196:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788576159.127199:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.127201:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576159.127202:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.127203:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788576159.127205:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788576159.127208:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788576159.127211:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788576159.127213:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788576159.127218:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788576159.127226:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393263744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788576159.127230:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.127233:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576159.127236:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348712680 x1875444393263488/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.127243:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788576159.127245:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788576159.127248:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788576159.127251:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788576159.127253:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788576159.127254:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788576159.127257:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788576159.127260:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd6800.
02000000:00000001:3.0:1788576159.127263:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788576159.127265:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788576159.127268:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348712680.
00000100:00000001:3.0:1788576159.127271:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788576159.127272:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.127274:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576159.127277:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348713480 x1875444393263616/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.127284:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788576159.127285:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788576159.127287:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788576159.127289:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788576159.127290:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788576159.127291:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788576159.127293:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788576159.127295:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd5400.
02000000:00000001:3.0:1788576159.127297:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788576159.127298:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788576159.127300:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348713480.
00000100:00000001:3.0:1788576159.127302:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788576159.127303:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.127305:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576159.127308:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348710380 x1875444393263744/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576159.127313:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788576159.127314:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788576159.127316:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788576159.127318:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788576159.127319:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788576159.127320:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788576159.127322:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788576159.127324:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895372cd4000.
02000000:00000001:3.0:1788576159.127326:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788576159.127327:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788576159.127328:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348710380.
00000100:00000001:3.0:1788576159.127331:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788576159.127332:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.127334:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576159.127341:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788576159.127343:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576159.127345:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788576159.127347:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576159.509557:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576159.509564:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576159.509566:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576159.509568:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576159.509572:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576159.509573:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576159.509574:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576159.509576:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576159.509578:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576159.509579:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576159.509588:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576159.509589:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576159.509672:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576159.509678:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576159.509683:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576159.637515:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576159.637523:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576159.637526:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576159.637531:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576159.637535:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576159.637540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576159.637543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576159.637545:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576159.637548:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576160.085525:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307763642, at 4307763648
00000100:00000010:0.0:1788576160.085540:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d840.
00000100:00000040:1.0:1788576160.088359:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307763642, at 4307763650
00000100:00000010:1.0:1788576160.088376:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91db40.
00010000:00000001:3.0:1788576160.533738:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576160.533745:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576160.533751:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576160.533752:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576160.533756:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576160.533757:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576160.533930:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576160.533934:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576160.533937:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576160.661886:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576160.661892:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576160.661895:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576160.661900:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576160.661904:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576160.661908:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576160.661910:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576160.661912:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576160.661916:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576161.557532:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576161.557540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576161.557546:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576161.557547:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576161.557551:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576161.557552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576161.558246:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576161.558252:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576161.558255:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576161.686324:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576161.686333:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576161.686335:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576161.686341:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576161.686344:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576161.686349:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576161.686352:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576161.686354:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576161.686357:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576162.581525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576162.581532:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576162.581538:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576162.581540:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576162.581543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576162.581544:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576162.581752:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576162.581756:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576162.581759:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576162.709517:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576162.709525:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576162.709527:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576162.709532:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576162.709536:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576162.709540:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576162.709543:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576162.709546:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576162.709549:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576163.607240:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576163.607247:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576163.607254:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576163.607255:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576163.607260:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576163.607261:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576163.607361:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576163.607364:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576163.607367:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576163.733635:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576163.733644:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576163.733647:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576163.733652:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576163.733656:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576163.733660:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576163.733663:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576163.733665:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576163.733668:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576164.245508:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576164.245519:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1788576164.245521:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576164.245525:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576164.245530:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576164.245537:0:186668:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13095
00000400:00000200:1.0:1788576164.245546:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576164.245552:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576164.245556:0:186668:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576164.245559:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245562:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576164.245565:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245567:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576164.245570:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576164.245572:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:1.0:1788576164.245575:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576164.245578:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:1.0:1788576164.245580:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:1.0:1788576164.245583:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:1.0:1788576164.245586:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245588:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576164.245589:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245591:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576164.245594:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:1.0:1788576164.245597:0:186668:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245600:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576164.245602:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576164.245605:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576164.245608:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576164.245610:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1788576164.245612:0:186668:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576164.245630:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7100.
00000020:00000040:1.0:1788576164.245634:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1788576164.245637:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576164.245639:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576164.245642:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576164.245644:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576164.245648:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576164.245655:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379758c00.
00000100:00000001:1.0:1788576164.245662:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576164.245671:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576164.245681:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576164.245686:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576164.245690:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086cb6a56] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576164.245714:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576164.245717:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576164.245720:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1788576164.245722:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576164.245724:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576164.245727:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576164.245729:0:186668:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13095
00000400:00000200:1.0:1788576164.245733:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576164.245736:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576164.245737:0:186668:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576164.245740:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245742:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576164.245744:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245745:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576164.245747:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576164.245748:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:1.0:1788576164.245750:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576164.245752:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:1.0:1788576164.245754:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:1.0:1788576164.245756:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:1.0:1788576164.245759:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245761:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576164.245762:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245764:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576164.245766:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:1.0:1788576164.245775:0:186668:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245778:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576164.245779:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576164.245785:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576164.245787:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576164.245789:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576164.245792:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1788576164.245795:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576164.245797:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576164.245802:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f7b80.
00000020:00000040:1.0:1788576164.245804:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:1.0:1788576164.245806:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576164.245807:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576164.245809:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576164.245810:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576164.245812:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576164.245816:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759c00.
00000100:00000001:1.0:1788576164.245819:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576164.245824:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576164.245831:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576164.245833:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576164.245836:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000193a2615] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576164.245844:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1788576164.245846:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1788576164.245848:0:186668:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1788576164.245849:0:186668:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1788576164.245851:0:186668:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1788576164.245853:0:186668:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:1.0:1788576164.245855:0:186668:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13095
00000400:00000200:1.0:1788576164.245859:0:186668:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1788576164.245861:0:186668:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1788576164.245863:0:186668:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1788576164.245866:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245868:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576164.245870:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245871:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:1.0:1788576164.245873:0:186668:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1788576164.245875:0:186668:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:1.0:1788576164.245877:0:186668:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1788576164.245878:0:186668:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:1.0:1788576164.245880:0:186668:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:1.0:1788576164.245882:0:186668:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:1.0:1788576164.245884:0:186668:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245886:0:186668:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1788576164.245888:0:186668:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245889:0:186668:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:1.0:1788576164.245891:0:186668:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:1.0:1788576164.245894:0:186668:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1788576164.245896:0:186668:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1788576164.245898:0:186668:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1788576164.245900:0:186668:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1788576164.245902:0:186668:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576164.245903:0:186668:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1788576164.245905:0:186668:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:1.0:1788576164.245908:0:186668:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576164.245909:0:186668:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1788576164.245912:0:186668:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8953796f4e00.
00000020:00000040:1.0:1788576164.245915:0:186668:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:1.0:1788576164.245917:0:186668:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1788576164.245918:0:186668:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1788576164.245920:0:186668:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1788576164.245921:0:186668:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1788576164.245923:0:186668:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1788576164.245926:0:186668:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895379759800.
00000100:00000001:1.0:1788576164.245929:0:186668:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576164.245933:0:186668:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1788576164.245940:0:186668:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1788576164.245942:0:186668:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576164.245946:0:186668:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086a12461] to pc [ptlrpcd_rcv+-1]  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1788576164.245952:0:186668:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576164.245954:0:186668:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1788576164.245958:0:186668:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:1.0:1788576164.245962:0:186668:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13105)
00000100:00000001:3.0:1788576164.246054:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788576164.246059:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788576164.246062:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788576164.246070:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246079:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576164.246081:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576164.246085:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576164.246087:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576164.246090:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576164.246094:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393263872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788576164.246098:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576164.246103:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576164.246106:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576164.246109:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788576164.246113:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788576164.246115:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576164.246121:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788576164.246125:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788576164.246131:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788576164.246137:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393263872, portal 25
00000100:00000001:3.0:1788576164.246139:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576164.246141:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345478400 : -130483364073216 : ffff8953796f7100)
00000100:00000040:3.0:1788576164.246147:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246155:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788576164.246157:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788576164.246160:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788576164.246164:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393263872, offset 0
00000400:00000200:3.0:1788576164.246170:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576164.246177:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576164.246184:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788576164.246190:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788576164.246193:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788576164.246197:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788576164.246200:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788576164.246204:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331317:331317:256:4294967295] 192.168.202.110@tcp LPNI seq info [331274:331274:8:4294967295]
00000400:00000200:3.0:1788576164.246209:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576164.246219:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788576164.246225:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576164.246230:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788576164.246237:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788576164.246240:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788576164.246246:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788576164.246249:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788576164.246254:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576164.246257:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788576164.246263:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788576164.246268:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788576164.246273:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788576164.246275:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788576164.246279:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788576164.246283:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246294:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576164.246297:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246304:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576164.246306:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788576164.246312:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576164.246316:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576164.246318:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307768761
00000100:00000010:3.0:1788576164.246323:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d720.
00000100:00000001:3.0:1788576164.246330:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788576164.246335:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307768761
00000100:00000010:3.0:1788576164.246338:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d1e0.
00000100:00000001:3.0:1788576164.246341:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788576164.246344:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788576164.246347:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576164.246350:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788576164.246354:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576164/real 1788576164]  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788576164.246364:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788576164.246366:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788576164.246369:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788576164.246371:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788576164.246373:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788576164.246376:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788576164.246380:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788576164.246389:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246418:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788576164.246420:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.246422:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576164.246425:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788576164.246429:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246435:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788576164.246439:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246445:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788576164.246448:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.246449:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576164.246451:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.246453:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788576164.246455:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788576164.246457:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788576164.246461:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788576164.246463:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788576164.246467:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788576164.246476:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393263872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788576164.246480:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788576164.246482:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246494:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576164.246495:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576164.246497:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576164.246499:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576164.246501:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576164.246504:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788576164.246508:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576164.246510:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576164.246511:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576164.246513:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788576164.246516:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788576164.246518:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576164.246522:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788576164.246524:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788576164.246527:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788576164.246530:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393264000, portal 10
00000100:00000001:3.0:1788576164.246532:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576164.246533:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345481088 : -130483364070528 : ffff8953796f7b80)
00000100:00000040:3.0:1788576164.246540:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246547:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788576164.246549:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788576164.246551:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788576164.246553:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393264000, offset 0
00000400:00000200:3.0:1788576164.246557:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576164.246561:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576164.246565:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788576164.246567:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788576164.246569:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788576164.246572:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788576164.246574:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788576164.246577:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331318:331318:256:4294967295] 192.168.202.110@tcp LPNI seq info [331275:331275:8:4294967295]
00000400:00000200:3.0:1788576164.246581:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576164.246588:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788576164.246591:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576164.246595:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788576164.246599:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788576164.246601:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788576164.246606:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788576164.246608:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788576164.246612:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576164.246614:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788576164.246616:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788576164.246619:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788576164.246680:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788576164.246682:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788576164.246685:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788576164.246689:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246698:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576164.246701:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246707:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576164.246708:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788576164.246711:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576164.246715:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788576164.246718:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576164.246720:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307767847
00000100:00000001:3.0:1788576164.246723:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307768761
00000100:00000001:3.0:1788576164.246725:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307767847
00000100:00000001:3.0:1788576164.246727:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307768761
00000100:00000001:3.0:1788576164.246729:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788576164.246732:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576164.246734:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788576164.246737:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576164/real 1788576164]  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788576164.246744:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788576164.246746:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788576164.246747:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788576164.246749:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788576164.246751:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788576164.246753:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788576164.246756:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788576164.246764:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246771:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788576164.246772:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.246774:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576164.246775:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788576164.246779:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246784:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788576164.246787:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246794:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788576164.246797:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.246798:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576164.246800:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.246801:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788576164.246803:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788576164.246805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788576164.246808:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788576164.246810:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788576164.246814:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788576164.246821:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788576164.246825:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788576164.246829:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246835:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576164.246837:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576164.246839:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576164.246840:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576164.246842:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576164.246844:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264128:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788576164.246847:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576164.246849:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576164.246850:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576164.246852:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788576164.246855:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788576164.246857:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576164.246860:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788576164.246862:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788576164.246865:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788576164.246868:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393264128, portal 4
00000100:00000001:3.0:1788576164.246870:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576164.246871:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590345469440 : -130483364082176 : ffff8953796f4e00)
00000100:00000040:3.0:1788576164.246876:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246882:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788576164.246885:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788576164.246886:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788576164.246889:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393264128, offset 0
00000400:00000200:3.0:1788576164.246893:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576164.246896:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576164.246899:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788576164.246902:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788576164.246904:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788576164.246906:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788576164.246909:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788576164.246912:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331319:331319:256:4294967295] 192.168.202.110@tcp LPNI seq info [331276:331276:8:4294967295]
00000400:00000200:3.0:1788576164.246916:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576164.246921:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788576164.246926:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576164.246929:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788576164.246932:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788576164.246935:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1788576164.246938:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788576164.246941:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1788576164.246945:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576164.246947:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788576164.246950:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788576164.246953:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788576164.246955:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788576164.246957:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788576164.246959:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788576164.246963:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246970:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576164.246973:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.246979:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576164.246980:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788576164.246983:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576164.246986:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788576164.246989:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576164.246990:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307767993
00000100:00000001:3.0:1788576164.246992:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307768761
00000100:00000001:3.0:1788576164.246994:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307767993
00000100:00000001:3.0:1788576164.246996:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307768761
00000100:00000001:3.0:1788576164.246998:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788576164.247000:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576164.247002:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788576164.247005:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576164/real 1788576164]  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788576164.247013:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788576164.247025:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788576164.247027:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788576164.247030:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788576164.247032:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788576164.247034:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788576164.247037:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788576164.247044:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.247050:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788576164.247051:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.247053:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576164.247054:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788576164.247057:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.247063:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788576164.247066:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.247073:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788576164.247075:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.247077:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576164.247078:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.247080:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788576164.247081:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788576164.247083:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788576164.247087:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788576164.247089:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788576164.247093:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788576164.247101:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff8953796f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264128:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788576164.247105:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.247107:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576164.247110:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7100 x1875444393263872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.247116:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788576164.247118:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788576164.247121:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788576164.247124:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788576164.247126:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788576164.247128:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788576164.247130:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788576164.247132:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379758c00.
02000000:00000001:3.0:1788576164.247136:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788576164.247138:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788576164.247140:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7100.
00000100:00000001:3.0:1788576164.247143:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788576164.247145:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.247147:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576164.247150:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f7b80 x1875444393264000/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.247156:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788576164.247157:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788576164.247159:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788576164.247161:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788576164.247162:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788576164.247163:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788576164.247165:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788576164.247167:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759c00.
02000000:00000001:3.0:1788576164.247169:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788576164.247170:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788576164.247171:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f7b80.
00000100:00000001:3.0:1788576164.247173:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788576164.247174:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.247175:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576164.247179:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff8953796f4e00 x1875444393264128/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576164.247185:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788576164.247186:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788576164.247187:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788576164.247190:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788576164.247191:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788576164.247192:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788576164.247194:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788576164.247196:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895379759800.
02000000:00000001:3.0:1788576164.247198:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788576164.247199:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788576164.247200:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8953796f4e00.
00000100:00000001:3.0:1788576164.247202:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788576164.247204:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.247206:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576164.247213:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788576164.247215:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576164.247217:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788576164.247219:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576164.629514:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576164.629520:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576164.629525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576164.629526:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576164.629529:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576164.629530:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576164.629616:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576164.629620:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576164.629670:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576164.757534:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576164.757543:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576164.757545:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576164.757552:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576164.757555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576164.757559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576164.757561:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576164.757564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576164.757567:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1788576165.205519:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307768761, at 4307768768
00000100:00000010:0.0:1788576165.205535:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d720.
00000100:00000040:1.0:1788576165.205614:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307768761, at 4307768768
00000100:00000010:1.0:1788576165.205619:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d1e0.
00010000:00000001:3.0:1788576165.653727:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576165.653733:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576165.653738:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576165.653739:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576165.653742:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576165.653744:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576165.658692:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576165.658698:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576165.658702:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576165.781558:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576165.781565:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576165.781567:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576165.781572:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576165.781576:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576165.781580:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576165.781582:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576165.781584:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576165.781587:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576166.679201:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576166.679210:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576166.679217:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576166.679218:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576166.679222:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576166.679223:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576166.680041:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576166.680045:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576166.680048:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576166.805519:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576166.805528:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576166.805530:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576166.805536:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576166.805540:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576166.805544:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576166.805546:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576166.805550:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576166.805552:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576167.702000:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576167.702008:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576167.702013:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576167.704700:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576167.704707:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576167.704711:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576167.704716:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576167.704718:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576167.704720:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576167.704725:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576167.704735:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576167.704738:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576167.704739:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576167.704741:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576167.704743:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576167.704744:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576167.704745:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576167.704747:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576167.705279:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576167.705283:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576167.705286:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576167.830013:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576167.830018:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576167.830020:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576167.830055:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576167.830060:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576167.830063:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576167.830066:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576167.830069:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576167.830071:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576168.726012:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576168.726020:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576168.726025:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576168.726058:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576168.726062:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576168.726063:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576168.726307:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576168.726310:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576168.726314:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576168.858152:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576168.858161:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576168.858163:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576168.858168:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576168.858171:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576168.858176:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576168.858178:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576168.858181:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576168.858184:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576169.366086:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576169.366095:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1788576169.366098:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576169.366101:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576169.366104:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576169.366111:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13100
00000400:00000200:2.0:1788576169.366119:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576169.366125:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576169.366127:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576169.366131:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366134:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576169.366136:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366139:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576169.366143:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576169.366145:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:2.0:1788576169.366147:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576169.366150:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:2.0:1788576169.366152:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:2.0:1788576169.366155:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:2.0:1788576169.366158:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366160:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576169.366162:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366164:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576169.366166:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:2.0:1788576169.366169:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366172:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576169.366175:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576169.366178:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576169.366180:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576169.366182:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1788576169.366185:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576169.366193:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e01180.
00000020:00000040:2.0:1788576169.366197:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1788576169.366201:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576169.366203:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576169.366205:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576169.366208:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576169.366211:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576169.366218:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10800.
00000100:00000001:2.0:1788576169.366225:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576169.366234:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576169.366244:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576169.366248:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576169.366253:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b74282d7] to pc [ptlrpcd_rcv+-1]  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576169.366277:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576169.366280:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576169.366283:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1788576169.366285:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576169.366286:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576169.366289:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576169.366292:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13100
00000400:00000200:2.0:1788576169.366295:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576169.366298:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576169.366300:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576169.366303:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366305:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576169.366307:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366310:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576169.366312:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576169.366313:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:2.0:1788576169.366314:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576169.366317:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:2.0:1788576169.366319:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:2.0:1788576169.366321:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:2.0:1788576169.366324:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366325:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576169.366327:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366329:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576169.366331:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:2.0:1788576169.366334:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366336:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576169.366338:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576169.366343:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576169.366345:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576169.366347:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576169.366350:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1788576169.366353:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576169.366355:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576169.366358:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e03480.
00000020:00000040:2.0:1788576169.366360:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:2.0:1788576169.366362:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576169.366364:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576169.366365:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576169.366367:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576169.366368:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576169.366372:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f11000.
00000100:00000001:2.0:1788576169.366375:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576169.366379:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576169.366387:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576169.366389:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576169.366417:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9656041] to pc [ptlrpcd_rcv+-1]  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576169.366433:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1788576169.366435:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1788576169.366439:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1788576169.366440:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1788576169.366442:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1788576169.366444:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:2.0:1788576169.366446:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13100
00000400:00000200:2.0:1788576169.366450:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576169.366452:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1788576169.366454:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1788576169.366456:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366458:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576169.366460:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366462:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:2.0:1788576169.366464:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1788576169.366466:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:2.0:1788576169.366467:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1788576169.366469:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:2.0:1788576169.366471:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:2.0:1788576169.366472:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:2.0:1788576169.366475:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366477:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1788576169.366479:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366480:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:2.0:1788576169.366482:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:2.0:1788576169.366485:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1788576169.366487:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576169.366489:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1788576169.366492:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576169.366493:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576169.366495:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1788576169.366497:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:2.0:1788576169.366500:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576169.366501:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1788576169.366506:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff895348e02300.
00000020:00000040:2.0:1788576169.366508:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:2.0:1788576169.366510:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1788576169.366511:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576169.366513:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1788576169.366514:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1788576169.366516:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1788576169.366519:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895349f10400.
00000100:00000001:2.0:1788576169.366522:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576169.366525:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1788576169.366533:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1788576169.366535:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576169.366539:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d274707a] to pc [ptlrpcd_rcv+-1]  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576169.366546:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576169.366548:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1788576169.366552:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1788576169.366556:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13110)
00000100:00000001:3.0:1788576169.366615:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788576169.366618:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1788576169.366621:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788576169.366627:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.366636:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576169.366638:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576169.366641:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576169.366644:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576169.366646:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576169.366650:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393264256:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788576169.366654:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576169.366657:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576169.366660:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576169.366662:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788576169.366667:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788576169.366669:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576169.366674:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788576169.366678:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788576169.366685:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788576169.366691:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393264256, portal 25
00000100:00000001:3.0:1788576169.366694:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576169.366696:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530775936 : -130484178775680 : ffff895348e01180)
00000100:00000040:3.0:1788576169.366701:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.366708:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788576169.366711:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788576169.366714:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788576169.366717:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393264256, offset 0
00000400:00000200:3.0:1788576169.366724:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576169.366730:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576169.366737:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788576169.366741:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788576169.366745:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788576169.366749:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788576169.366752:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788576169.366756:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331320:331320:256:4294967295] 192.168.202.110@tcp LPNI seq info [331277:331277:8:4294967295]
00000400:00000200:3.0:1788576169.366761:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576169.366770:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788576169.366776:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576169.366781:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788576169.366787:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788576169.366791:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788576169.366796:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788576169.366799:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1788576169.366803:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576169.366806:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788576169.366812:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788576169.366817:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788576169.366822:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788576169.366824:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788576169.366828:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788576169.366832:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.366841:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576169.366844:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.366852:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576169.366854:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788576169.366860:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576169.366863:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576169.366866:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307773882
00000100:00000010:3.0:1788576169.366871:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91d120.
00000100:00000001:3.0:1788576169.366877:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1788576169.366895:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307773882
00000100:00000010:3.0:1788576169.366898:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff89537c91de40.
00000100:00000001:3.0:1788576169.366901:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1788576169.366904:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788576169.366907:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576169.366909:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788576169.366914:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576169/real 1788576169]  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788576169.366924:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788576169.366925:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788576169.366927:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788576169.366930:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788576169.366932:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788576169.366934:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788576169.366938:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788576169.366947:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.366953:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788576169.366955:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369108:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576169.369115:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788576169.369129:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369139:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788576169.369145:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369152:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788576169.369156:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369158:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576169.369159:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369162:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788576169.369164:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788576169.369168:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788576169.369172:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788576169.369174:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788576169.369178:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788576169.369186:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e01180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393264256:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1788576169.369191:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788576169.369195:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369202:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576169.369204:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576169.369207:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576169.369210:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576169.369213:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576169.369216:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264384:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788576169.369220:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576169.369224:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576169.369227:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576169.369229:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788576169.369237:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788576169.369239:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576169.369246:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788576169.369250:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788576169.369257:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788576169.369263:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393264384, portal 10
00000100:00000001:3.0:1788576169.369265:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576169.369267:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530784896 : -130484178766720 : ffff895348e03480)
00000100:00000040:3.0:1788576169.369273:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369280:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788576169.369283:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788576169.369286:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788576169.369289:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393264384, offset 0
00000400:00000200:3.0:1788576169.369294:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576169.369302:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576169.369309:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788576169.369314:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788576169.369317:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788576169.369321:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788576169.369323:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788576169.369327:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331321:331321:256:4294967295] 192.168.202.110@tcp LPNI seq info [331278:331278:8:4294967295]
00000400:00000200:3.0:1788576169.369331:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576169.369341:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788576169.369347:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576169.369351:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788576169.369356:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788576169.369360:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788576169.369365:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788576169.369368:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1788576169.369372:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576169.369375:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788576169.369381:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788576169.369385:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788576169.369405:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788576169.369407:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788576169.369411:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788576169.369415:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369431:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576169.369435:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369442:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576169.369444:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788576169.369450:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576169.369454:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:3.0:1788576169.369457:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576169.369460:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307772970
00000100:00000001:3.0:1788576169.369464:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307773882
00000100:00000001:3.0:1788576169.369466:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307772970
00000100:00000001:3.0:1788576169.369469:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307773882
00000100:00000001:3.0:1788576169.369471:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788576169.369474:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576169.369476:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788576169.369480:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576169/real 1788576169]  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788576169.369489:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788576169.369490:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788576169.369492:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788576169.369494:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788576169.369496:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788576169.369498:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788576169.369502:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788576169.369509:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369517:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788576169.369519:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369520:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576169.369522:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788576169.369525:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369532:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788576169.369536:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369543:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788576169.369545:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369547:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576169.369548:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369550:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788576169.369552:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788576169.369555:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788576169.369558:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788576169.369560:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788576169.369564:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788576169.369572:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264384:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1788576169.369576:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1788576169.369580:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369587:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1788576169.369588:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576169.369590:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1788576169.369592:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576169.369594:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576169.369597:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264512:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788576169.369600:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1788576169.369602:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1788576169.369604:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576169.369606:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1788576169.369609:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788576169.369611:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1788576169.369614:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1788576169.369616:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89534b6c5688.
00000400:00000010:3.0:1788576169.369619:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b65263b8.
00000100:00000200:3.0:1788576169.369622:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393264512, portal 4
00000100:00000001:3.0:1788576169.369623:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1788576169.369625:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613589530780416 : -130484178771200 : ffff895348e02300)
00000100:00000040:3.0:1788576169.369629:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369634:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1788576169.369636:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1788576169.369638:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8952b6526908.
00000100:00000200:3.0:1788576169.369641:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393264512, offset 0
00000400:00000200:3.0:1788576169.369644:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1788576169.369648:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576169.369651:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1788576169.369654:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788576169.369656:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1788576169.369658:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788576169.369660:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1788576169.369663:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331322:331322:256:4294967295] 192.168.202.110@tcp LPNI seq info [331279:331279:8:4294967295]
00000400:00000200:3.0:1788576169.369668:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576169.369674:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788576169.369678:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576169.369681:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534728b000.
00000800:00000200:3.0:1788576169.369685:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1788576169.369688:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1788576169.369692:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1788576169.369694:0:2361:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1788576169.369698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1788576169.369700:0:2361:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff89534728b000.
00000400:00000200:3.0:1788576169.369703:0:2361:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1788576169.369707:0:2361:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1788576169.369709:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b6526908
00000400:00000010:3.0:1788576169.369711:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b6526908.
00000100:00000001:3.0:1788576169.369713:0:2361:0:(events.c:35:request_out_callback()) Process entered
00000100:00000200:3.0:1788576169.369717:0:2361:0:(events.c:40:request_out_callback()) @@@ type 5, status -5  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369725:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576169.369728:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369734:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576169.369736:0:2361:0:(events.c:69:request_out_callback()) Process leaving
00000100:00000001:3.0:1788576169.369738:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576169.369741:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:3.0:1788576169.369744:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1788576169.369746:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307773116
00000100:00000001:3.0:1788576169.369749:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307773882
00000100:00000001:3.0:1788576169.369750:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307773116
00000100:00000001:3.0:1788576169.369753:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307773882
00000100:00000001:3.0:1788576169.369755:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1788576169.369757:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576169.369759:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1788576169.369762:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788576169/real 1788576169]  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1788576169.369770:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1788576169.369771:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1788576169.369773:0:2361:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff89534b6c5688.
00000400:00000200:3.0:1788576169.369775:0:2361:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8952b65263b8
00000400:00000010:3.0:1788576169.369776:0:2361:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8952b65263b8.
00000100:00000001:3.0:1788576169.369779:0:2361:0:(events.c:80:reply_in_callback()) Process entered
00000100:00000200:3.0:1788576169.369782:0:2361:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788576169.369791:0:2361:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369797:0:2361:0:(events.c:165:reply_in_callback()) Process leaving
00000100:00000001:3.0:1788576169.369798:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369800:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576169.369801:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1788576169.369805:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369812:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1788576169.369816:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369821:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1788576169.369823:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369824:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1788576169.369825:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369827:0:2361:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1788576169.369829:0:2361:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1788576169.369832:0:2361:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1788576169.369835:0:2361:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1788576169.369837:0:2361:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1788576169.369841:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1788576169.369855:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff895348e02300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264512:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1788576169.369859:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369861:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576169.369864:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e01180 x1875444393264256/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369871:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788576169.369873:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788576169.369875:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cf8400.
02000000:00000001:3.0:1788576169.369878:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788576169.369881:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788576169.369882:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372d14000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1788576169.369885:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788576169.369887:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10800.
02000000:00000001:3.0:1788576169.369890:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788576169.369892:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788576169.369894:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e01180.
00000100:00000001:3.0:1788576169.369897:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788576169.369899:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369900:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576169.369903:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e03480 x1875444393264384/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369910:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788576169.369911:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788576169.369912:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfac00.
02000000:00000001:3.0:1788576169.369914:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788576169.369915:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788576169.369917:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dae800 refcount=9 obd=lustre-MDT0000-mdc-ffff89524108d000
00000020:00000001:3.0:1788576169.369918:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788576169.369919:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f11000.
02000000:00000001:3.0:1788576169.369921:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788576169.369922:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788576169.369923:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e03480.
00000100:00000001:3.0:1788576169.369926:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788576169.369927:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369929:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1788576169.369932:0:2361:0:(client.c:2876:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff895348e02300 x1875444393264512/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1788576194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576169.369939:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1788576169.369940:0:2361:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1788576169.369941:0:2361:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff895372cfb800.
02000000:00000001:3.0:1788576169.369943:0:2361:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1788576169.369944:0:2361:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1788576169.369945:0:2361:0:(genops.c:1223:class_import_put()) import ffff895372dab800 refcount=3 obd=lustre-OST0000-osc-ffff89524108d000
00000020:00000001:3.0:1788576169.369947:0:2361:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1788576169.369948:0:2361:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff895349f10400.
02000000:00000001:3.0:1788576169.369950:0:2361:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1788576169.369951:0:2361:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1788576169.369952:0:2361:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff895348e02300.
00000100:00000001:3.0:1788576169.369953:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1788576169.369955:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369957:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1788576169.369963:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1788576169.369965:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576169.369968:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1788576169.369970:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576169.749588:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576169.749597:0:216524:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1788576169.749601:0:216524:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1788576169.749604:0:216524:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576169.749610:0:216524:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1788576169.749612:0:216524:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576169.749614:0:216524:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576169.749617:0:216524:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576169.749620:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576169.749622:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576169.749626:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576169.749627:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576169.749748:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576169.749752:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576169.749755:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576169.877657:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576169.877665:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576169.877668:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576169.877673:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576169.877677:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576169.877681:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576169.877683:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576169.877686:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576169.877689:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1788576170.325473:0:216397:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744110 (cpu 1) has reached its deadline 4307773882, at 4307773888
00000100:00000010:1.0:1788576170.325490:0:216397:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91de40.
00000100:00000040:0.0:1788576170.325529:0:216538:0:(connection.c:75:cpu_latency_work()) work item of ffff895350744000 (cpu 0) has reached its deadline 4307773882, at 4307773888
00000100:00000010:0.0:1788576170.325536:0:216538:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff89537c91d120.
00010000:00000001:3.0:1788576170.773525:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576170.773533:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576170.773539:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576170.773552:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576170.773556:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576170.773557:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576170.773707:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576170.773711:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576170.773714:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576170.901526:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576170.901535:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576170.901537:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576170.901551:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576170.901555:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576170.901559:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576170.901562:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576170.901564:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576170.901567:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576171.797510:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576171.797516:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576171.797520:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576171.797521:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576171.797523:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576171.797524:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576171.801696:0:190931:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576171.801703:0:190931:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576171.801710:0:190931:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1788576171.925732:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1788576171.925740:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1788576171.925743:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1788576171.925748:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:1.0:1788576171.925752:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1788576171.925756:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576171.925759:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1788576171.925761:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1788576171.925765:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1788576172.438128:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8953539a89c0] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:0.0:1788576172.438137:0:2353:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:0.0:1788576172.438143:0:2353:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff895353ad1c00 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:1.0:1788576172.439350:0:2354:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1788576172.439357:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788576172.439367:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1788576172.439372:0:2354:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1788576172.439375:0:2354:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1788576172.821732:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576172.821739:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576172.821745:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576172.821746:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576172.821750:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576172.821751:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1788576172.821826:0:179192:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1788576172.821828:0:179192:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1788576172.821831:0:179192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576172.952038:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576172.952046:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576172.952048:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576172.952055:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576172.952058:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576172.952062:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576172.952064:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576172.952067:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576172.952070:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576173.845543:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576173.845549:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576173.845555:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576173.845556:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1788576173.845559:0:216524:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1788576173.845560:0:216524:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1788576173.848831:0:185619:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1788576173.848840:0:185619:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1788576173.848846:0:185619:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1788576173.973549:0:216043:0:(client.c:2568:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1788576173.973558:0:216043:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1788576173.973560:0:216043:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1788576173.973566:0:216043:0:(client.c:2716:ptlrpc_set_wait()) set ffff89534b329000 going to sleep for 0 seconds
00000100:00000001:0.0:1788576173.973570:0:216043:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1788576173.973575:0:216043:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576173.973578:0:216043:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1788576173.973580:0:216043:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1788576173.973583:0:216043:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576174.485620:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) cf421482-2d8c-4284-9ac1-5ab3cd8c702c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576174.485631:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1788576174.485634:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576174.485638:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372d14000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576174.485643:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576174.485650:0:188751:0:(import.c:545:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 13105
00000400:00000200:3.0:1788576174.485659:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576174.485666:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576174.485669:0:188751:0:(import.c:630:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576174.485672:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485675:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576174.485678:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485680:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576174.485685:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576174.485687:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711eb4
00000020:00000001:3.0:1788576174.485690:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576174.485693:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d10800 refcount=3
00000020:00000001:3.0:1788576174.485696:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000020:00000001:3.0:1788576174.485698:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234433536 : -130483475118080 : ffff895372d10800)
00000100:00000040:3.0:1788576174.485701:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485703:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576174.485705:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485707:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576174.485710:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d10800 : new refcount 2
00000100:00080000:3.0:1788576174.485714:0:188751:0:(import.c:665:import_select_connection()) MGC192.168.202.110@tcp: import ffff895372d14000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485716:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576174.485719:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576174.485723:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576174.485726:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576174.485728:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1788576174.485731:0:188751:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576174.485743:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0380.
00000020:00000040:3.0:1788576174.485748:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372d14000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1788576174.485752:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576174.485754:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576174.485757:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576174.485760:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576174.485765:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576174.485773:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfb800.
00000100:00000001:3.0:1788576174.485781:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576174.485791:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0380 x1875444393264640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576174.485802:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576174.485807:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576174.485811:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000056aca9bc] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0380 x1875444393264640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576174.485842:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576174.485845:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576174.485849:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1788576174.485850:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576174.485852:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dae800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576174.485855:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576174.485858:0:188751:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13105
00000400:00000200:3.0:1788576174.485862:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576174.485865:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576174.485867:0:188751:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576174.485870:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485872:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576174.485874:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485876:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576174.485878:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576174.485879:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f01
00000020:00000001:3.0:1788576174.485881:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576174.485883:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff89534889e000 refcount=5
00000020:00000001:3.0:1788576174.485885:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000020:00000001:3.0:1788576174.485887:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613589525127168 : -130484184424448 : ffff89534889e000)
00000100:00000040:3.0:1788576174.485890:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485892:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576174.485894:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485896:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576174.485898:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff89534889e000 : new refcount 4
00000100:00080000:3.0:1788576174.485901:0:188751:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff89524108d000: import ffff895372dae800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576174.485903:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576174.485906:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576174.485911:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576174.485914:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576174.485916:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576174.485921:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1788576174.485923:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576174.485925:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576174.485929:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e1c00.
00000020:00000040:3.0:1788576174.485931:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dae800 refcount=10 obd=lustre-MDT0000-mdc-ffff89524108d000
02000000:00000001:3.0:1788576174.485933:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576174.485935:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576174.485936:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576174.485938:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576174.485939:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576174.485954:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cfac00.
00000100:00000001:3.0:1788576174.485958:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576174.485964:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e1c00 x1875444393264768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576174.486002:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576174.486004:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576174.486009:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005971625f] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e1c00 x1875444393264768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576174.486016:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1788576174.486018:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1788576174.486021:0:188751:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1788576174.486023:0:188751:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1788576174.486024:0:188751:0:(import.c:65:import_set_state_nolock()) ffff895372dab800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1788576174.486027:0:188751:0:(import.c:520:import_select_connection()) Process entered
00000100:00080000:3.0:1788576174.486030:0:188751:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: connect to NID 192.168.202.110@tcp last attempt 13105
00000400:00000200:3.0:1788576174.486034:0:188751:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1788576174.486036:0:188751:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1788576174.486038:0:188751:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1788576174.486041:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.486043:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576174.486044:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.486046:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000001:3.0:1788576174.486048:0:188751:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1788576174.486050:0:188751:0:(genops.c:804:class_conn2export()) looking for export cookie 0xdb00379715711f0f
00000020:00000001:3.0:1788576174.486051:0:188751:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1788576174.486053:0:188751:0:(lustre_handles.c:148:class_handle2object()) GET export ffff895372d14800 refcount=4
00000020:00000001:3.0:1788576174.486055:0:188751:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000020:00000001:3.0:1788576174.486057:0:188751:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446613590234449920 : -130483475101696 : ffff895372d14800)
00000100:00000040:3.0:1788576174.486059:0:188751:0:(lustre_net.h:1977:ptlrpc_connection_put()) PUT conn=ffff895344f01780 refcount 11 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.486061:0:188751:0:(connection.c:157:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1788576174.486062:0:188751:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff895344f01780 refcount 12 to 192.168.202.110@tcp
00000100:00000001:3.0:1788576174.486064:0:188751:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446613589464717184 : -130484244834432 : ffff895344f01780)
00000020:00000040:3.0:1788576174.486066:0:188751:0:(genops.c:879:class_export_put()) PUTting export ffff895372d14800 : new refcount 3
00000100:00080000:3.0:1788576174.486068:0:188751:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff89524108d000: import ffff895372dab800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1788576174.486070:0:188751:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1788576174.486072:0:188751:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1788576174.486075:0:188751:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1788576174.486077:0:188751:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576174.486078:0:188751:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1788576174.486080:0:188751:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 1703936, lost: 0.
00000008:00000001:3.0:1788576174.486082:0:188751:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576174.486083:0:188751:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1788576174.486087:0:188751:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff89536d1e0000.
00000020:00000040:3.0:1788576174.486089:0:188751:0:(genops.c:1210:class_import_get()) import ffff895372dab800 refcount=4 obd=lustre-OST0000-osc-ffff89524108d000
02000000:00000001:3.0:1788576174.486092:0:188751:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1788576174.486094:0:188751:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1788576174.486095:0:188751:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1788576174.486096:0:188751:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1788576174.486098:0:188751:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1788576174.486101:0:188751:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff895372cf8400.
00000100:00000001:3.0:1788576174.486105:0:188751:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1788576174.486110:0:188751:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff89536d1e0000 x1875444393264896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1788576174.486117:0:188751:0:(jobid.c:959:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1788576174.486119:0:188751:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576174.486122:0:188751:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b21b7285] to pc [ptlrpcd_rcv+-1]  req@ffff89536d1e0000 x1875444393264896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1788576174.486128:0:188751:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1788576174.486131:0:188751:0:(pinger.c:229:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1788576174.486135:0:188751:0:(pinger.c:247:ptlrpc_pinger_process_import()) c58e8409-3a6d-4b2b-8875-5894a3f266ca->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:3.0:1788576174.486139:0:188751:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13115)
00000100:00000001:2.0:1788576174.486199:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576174.486204:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576174.486207:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576174.486213:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0380 x1875444393264640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576174.486222:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576174.486225:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576174.486229:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576174.486231:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576174.486233:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576174.486237:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cf421482-2d8c-4284-9ac1-5ab3cd8c702c:2361:1875444393264640:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1788576174.486240:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576174.486245:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576174.486249:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576174.486251:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576174.486255:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f12c00.
02000000:00000001:2.0:1788576174.486257:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576174.486263:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576174.486268:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fb00.
00000400:00000010:2.0:1788576174.486276:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c656e8.
00000100:00000200:2.0:1788576174.486283:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393264640, portal 25
00000100:00000001:2.0:1788576174.486286:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576174.486288:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138815360 : -130483570736256 : ffff89536d1e0380)
00000100:00000040:2.0:1788576174.486293:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0380 x1875444393264640/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1788576199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576174.486299:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576174.486302:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576174.486305:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c657f8.
00000100:00000200:2.0:1788576174.486310:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875444393264640, offset 0
00000400:00000200:2.0:1788576174.486316:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576174.486324:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576174.486331:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576174.486336:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576174.486340:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576174.486344:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576174.486347:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576174.486351:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331323:331323:256:4294967295] 192.168.202.110@tcp LPNI seq info [331280:331280:8:4294967295]
00000400:00000200:2.0:1788576174.486355:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576174.486363:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576174.486370:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576174.486374:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133200.
00000800:00000200:2.0:1788576174.486380:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576174.486384:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788576174.486419:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576174.486425:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576174.486428:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4307779002
00000100:00000010:2.0:1788576174.486433:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c900.
00000100:00000001:2.0:1788576174.486442:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1788576174.486448:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4307779002
00000100:00000010:2.0:1788576174.486452:0:2361:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff895344f0c540.
00000100:00000001:2.0:1788576174.486457:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1788576174.486460:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576174.486463:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576174.486466:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576174.486469:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e1c00 x1875444393264768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576174.486477:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576174.486479:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576174.486481:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576174.486483:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576174.486485:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576174.486488:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1788576174.486490:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576174.486492:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576174.486494:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576174.486495:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576174.486500:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f13400.
02000000:00000001:2.0:1788576174.486502:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576174.486505:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576174.486507:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119f6e0.
00000400:00000010:2.0:1788576174.486510:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65660.
00000100:00000200:2.0:1788576174.486514:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393264768, portal 10
00000100:00000001:2.0:1788576174.486516:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576174.486517:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138821632 : -130483570729984 : ffff89536d1e1c00)
00000100:00000040:2.0:1788576174.486524:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e1c00 x1875444393264768/t0(0) o38->lustre-MDT0000-mdc-ffff89524108d000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1788576199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576174.486529:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576174.486531:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576174.486534:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65110.
00000100:00000200:2.0:1788576174.486536:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875444393264768, offset 0
00000400:00000200:2.0:1788576174.486541:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576174.486545:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576174.486548:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576174.486550:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576174.486552:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576174.486554:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576174.486556:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576174.486560:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331324:331324:255:4294967295] 192.168.202.110@tcp LPNI seq info [331281:331281:7:4294967295]
00000400:00000200:2.0:1788576174.486564:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576174.486571:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576174.486575:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576174.486579:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133c00.
00000800:00000200:2.0:1788576174.486583:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576174.486587:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788576174.486589:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576174.486592:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: using a timeout of 37413 usecs (38 jiffies)
00000400:00000200:2.0:1788576174.486595:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576174.486597:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307778087
00000100:00000001:2.0:1788576174.486600:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307779002
00000100:00000001:2.0:1788576174.486602:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307778087
00000100:00000001:2.0:1788576174.486604:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307779002
00000100:00000001:2.0:1788576174.486606:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576174.486608:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576174.486609:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1788576174.486613:0:2361:0:(lustre_net.h:2417:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff89536d1e0000 x1875444393264896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576174.486619:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1788576174.486620:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576174.486623:0:2361:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1788576174.486624:0:2361:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1788576174.486626:0:2361:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1788576174.486628:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff89536d1e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c58e8409-3a6d-4b2b-8875-5894a3f266ca:2361:1875444393264896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1788576174.486631:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1788576174.486632:0:2361:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1788576174.486634:0:2361:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1788576174.486635:0:2361:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1788576174.486639:0:2361:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff895349f11800.
02000000:00000001:2.0:1788576174.486641:0:2361:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1788576174.486644:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1788576174.486646:0:2361:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff89535119fdc0.
00000400:00000010:2.0:1788576174.486649:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c654c8.
00000100:00000200:2.0:1788576174.486652:0:2361:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875444393264896, portal 4
00000100:00000001:2.0:1788576174.486653:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1788576174.486654:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446613590138814464 : -130483570737152 : ffff89536d1e0000)
00000100:00000040:2.0:1788576174.486660:0:2361:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff89536d1e0000 x1875444393264896/t0(0) o8->lustre-OST0000-osc-ffff89524108d000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1788576199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1788576174.486666:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1788576174.486668:0:2361:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1788576174.486670:0:2361:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff895343c65f68.
00000100:00000200:2.0:1788576174.486672:0:2361:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875444393264896, offset 0
00000400:00000200:2.0:1788576174.486675:0:2361:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1788576174.486679:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576174.486682:0:2361:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1788576174.486684:0:2361:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788576174.486687:0:2361:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1788576174.486688:0:2361:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788576174.486691:0:2361:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1788576174.486694:0:2361:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [331325:331325:254:4294967295] 192.168.202.110@tcp LPNI seq info [331282:331282:6:4294967295]
00000400:00000200:2.0:1788576174.486698:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1788576174.486703:0:2361:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788576174.486707:0:2361:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1788576174.486711:0:2361:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff89534b133400.
00000800:00000200:2.0:1788576174.486714:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1788576174.486717:0:2361:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff89535096ae40] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1788576174.486720:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1788576174.486722:0:2361:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: using a timeout of 183067 usecs (184 jiffies)
00000400:00000200:2.0:1788576174.486725:0:2361:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1788576174.486727:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 0 is 4307778233
00000100:00000001:2.0:1788576174.486729:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 0, keep current deadline 4307779002
00000100:00000001:2.0:1788576174.486731:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS new deadline estimation for cpu 1 is 4307778233
00000100:00000001:2.0:1788576174.486733:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff89524108d000: PM QoS request active for cpu 1, keep current deadline 4307779002
00000100:00000001:2.0:1788576174.486734:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1788576174.486736:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576174.486739:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576174.486741:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1788576174.486748:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1788576174.486751:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1788576174.486753:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1788576174.486754:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1788576174.486755:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1788576174.486757:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
Debug log: 186292 lines, 186292 kept, 0 dropped, 0 bad.